diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json index 0528c4349..9e8207fe4 100644 --- a/.github/aw/actions-lock.json +++ b/.github/aw/actions-lock.json @@ -20,6 +20,11 @@ "version": "v7.0.0", "sha": "bbbca2ddaa5d8feaa63e36b76fdaad77386f024f" }, + "github/gh-aw/actions/setup@v0.50.5": { + "repo": "github/gh-aw/actions/setup", + "version": "v0.50.5", + "sha": "a7d371cc7e68f270ded0592942424548e05bf1c2" + }, "github/gh-aw/actions/setup@v0.52.1": { "repo": "github/gh-aw/actions/setup", "version": "v0.52.1", diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 40f762f67..804e6f0d4 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -7,25 +7,32 @@ updates: - package-ecosystem: 'github-actions' directory: '/' multi-ecosystem-group: 'all' + patterns: ['*'] - package-ecosystem: 'devcontainers' directory: '/' multi-ecosystem-group: 'all' + patterns: ['*'] # Node.js dependencies - package-ecosystem: 'npm' directory: '/nodejs' multi-ecosystem-group: 'all' + patterns: ['*'] - package-ecosystem: 'npm' directory: '/test/harness' multi-ecosystem-group: 'all' + patterns: ['*'] # Python dependencies - package-ecosystem: 'pip' directory: '/python' multi-ecosystem-group: 'all' + patterns: ['*'] # Go dependencies - package-ecosystem: 'gomod' directory: '/go' multi-ecosystem-group: 'all' + patterns: ['*'] # .NET dependencies - package-ecosystem: 'nuget' directory: '/dotnet' multi-ecosystem-group: 'all' + patterns: ['*'] diff --git a/.github/workflows/cross-repo-issue-analysis.lock.yml b/.github/workflows/cross-repo-issue-analysis.lock.yml new file mode 100644 index 000000000..423d3a045 --- /dev/null +++ b/.github/workflows/cross-repo-issue-analysis.lock.yml @@ -0,0 +1,1377 @@ +# +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ +# | _ |/ _` |/ _ \ '_ \| __| |/ __| +# | | | | (_| | __/ | | | |_| | (__ +# \_| |_/\__, |\___|_| |_|\__|_|\___| +# __/ | +# _ _ |___/ +# | | | | / _| | +# | | | | ___ _ __ _ __| |_| | _____ ____ +# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| +# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ +# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ +# +# This file was automatically generated by gh-aw (v0.50.5). DO NOT EDIT. +# +# To update this file, edit the corresponding .md file and run: +# gh aw compile +# Not all edits will cause changes to this file. +# +# For more information: https://github.github.com/gh-aw/introduction/overview/ +# +# Analyzes copilot-sdk issues to determine if a fix is needed in copilot-agent-runtime, then opens a linked issue and suggested-fix PR there +# +# gh-aw-metadata: {"schema_version":"v1","frontmatter_hash":"067feb8b9d0762a67a2601351b53b40d404cadfda759e2ae77b3c941f48568dc","compiler_version":"v0.50.5"} + +name: "SDK Runtime Triage" +"on": + issues: + types: + - opened + workflow_dispatch: + inputs: + issue_number: + description: Issue number to analyze + required: true + type: string + +permissions: {} + +concurrency: + group: "gh-aw-${{ github.workflow }}-${{ github.event.issue.number }}" + +run-name: "SDK Runtime Triage" + +jobs: + activation: + needs: pre_activation + if: needs.pre_activation.outputs.activated == 'true' + runs-on: ubuntu-slim + permissions: + contents: read + outputs: + body: ${{ steps.sanitized.outputs.body }} + comment_id: "" + comment_repo: "" + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + text: ${{ steps.sanitized.outputs.text }} + title: ${{ steps.sanitized.outputs.title }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@a7d371cc7e68f270ded0592942424548e05bf1c2 # v0.50.5 + with: + destination: /opt/gh-aw/actions + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Validate context variables + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/validate_context_variables.cjs'); + await main(); + - name: Checkout .github and .agents folders + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + .github + .agents + fetch-depth: 1 + persist-credentials: false + - name: Check workflow file timestamps + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_WORKFLOW_FILE: "cross-repo-issue-analysis.lock.yml" + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + - name: Compute current body text + id: sanitized + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/compute_text.cjs'); + await main(); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_EXPR_54492A5B: ${{ github.event.issue.number || inputs.issue_number }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_ISSUE_TITLE: ${{ github.event.issue.title }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + run: | + bash /opt/gh-aw/actions/create_prompt_first.sh + { + cat << 'GH_AW_PROMPT_EOF' + + GH_AW_PROMPT_EOF + cat "/opt/gh-aw/prompts/xpia.md" + cat "/opt/gh-aw/prompts/temp_folder_prompt.md" + cat "/opt/gh-aw/prompts/markdown.md" + cat "/opt/gh-aw/prompts/safe_outputs_prompt.md" + cat << 'GH_AW_PROMPT_EOF' + + Tools: add_comment, create_issue, create_pull_request, add_labels, missing_tool, missing_data + GH_AW_PROMPT_EOF + cat "/opt/gh-aw/prompts/safe_outputs_create_pull_request.md" + cat << 'GH_AW_PROMPT_EOF' + + + The following GitHub context information is available for this workflow: + {{#if __GH_AW_GITHUB_ACTOR__ }} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if __GH_AW_GITHUB_REPOSITORY__ }} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if __GH_AW_GITHUB_WORKSPACE__ }} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} + - **issue-number**: #__GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} + - **discussion-number**: #__GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} + - **pull-request-number**: #__GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} + - **comment-id**: __GH_AW_GITHUB_EVENT_COMMENT_ID__ + {{/if}} + {{#if __GH_AW_GITHUB_RUN_ID__ }} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' + {{#runtime-import .github/workflows/cross-repo-issue-analysis.md}} + GH_AW_PROMPT_EOF + } > "$GH_AW_PROMPT" + - name: Interpolate variables and render templates + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_54492A5B: ${{ github.event.issue.number || inputs.issue_number }} + GH_AW_GITHUB_EVENT_ISSUE_TITLE: ${{ github.event.issue.title }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + await main(); + - name: Substitute placeholders + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_54492A5B: ${{ github.event.issue.number || inputs.issue_number }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_ISSUE_TITLE: ${{ github.event.issue.title }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: ${{ needs.pre_activation.outputs.activated }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + + const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_EXPR_54492A5B: process.env.GH_AW_EXPR_54492A5B, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, + GH_AW_GITHUB_EVENT_ISSUE_TITLE: process.env.GH_AW_GITHUB_EVENT_ISSUE_TITLE, + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/validate_prompt_placeholders.sh + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/print_prompt_summary.sh + - name: Upload prompt artifact + if: success() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: prompt + path: /tmp/gh-aw/aw-prompts/prompt.txt + retention-days: 1 + + agent: + needs: activation + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_AW_ASSETS_ALLOWED_EXTS: "" + GH_AW_ASSETS_BRANCH: "" + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs.jsonl + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + GH_AW_WORKFLOW_ID_SANITIZED: crossrepoissueanalysis + outputs: + checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} + detection_success: ${{ steps.detection_conclusion.outputs.success }} + has_patch: ${{ steps.collect_output.outputs.has_patch }} + model: ${{ steps.generate_aw_info.outputs.model }} + output: ${{ steps.collect_output.outputs.output }} + output_types: ${{ steps.collect_output.outputs.output_types }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@a7d371cc7e68f270ded0592942424548e05bf1c2 # v0.50.5 + with: + destination: /opt/gh-aw/actions + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Create gh-aw temp directory + run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh + - name: Clone copilot-agent-runtime + run: git clone --depth 1 https://x-access-token:${{ secrets.RUNTIME_TRIAGE_TOKEN }}@github.com/github/copilot-agent-runtime.git ${{ github.workspace }}/copilot-agent-runtime + + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Checkout PR branch + id: checkout-pr + if: | + github.event.pull_request + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_TOKEN: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + with: + github-token: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/checkout_pr_branch.cjs'); + await main(); + - name: Generate agentic run info + id: generate_aw_info + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const fs = require('fs'); + + const awInfo = { + engine_id: "copilot", + engine_name: "GitHub Copilot CLI", + model: process.env.GH_AW_MODEL_AGENT_COPILOT || "", + version: "", + agent_version: "0.0.418", + cli_version: "v0.50.5", + workflow_name: "SDK Runtime Triage", + experimental: false, + supports_tools_allowlist: true, + run_id: context.runId, + run_number: context.runNumber, + run_attempt: process.env.GITHUB_RUN_ATTEMPT, + repository: context.repo.owner + '/' + context.repo.repo, + ref: context.ref, + sha: context.sha, + actor: context.actor, + event_name: context.eventName, + staged: false, + allowed_domains: ["defaults"], + firewall_enabled: true, + awf_version: "v0.23.0", + awmg_version: "v0.1.5", + steps: { + firewall: "squid" + }, + created_at: new Date().toISOString() + }; + + // Write to /tmp/gh-aw directory to avoid inclusion in PR + const tmpPath = '/tmp/gh-aw/aw_info.json'; + fs.writeFileSync(tmpPath, JSON.stringify(awInfo, null, 2)); + console.log('Generated aw_info.json at:', tmpPath); + console.log(JSON.stringify(awInfo, null, 2)); + + // Set model as output for reuse in other steps/jobs + core.setOutput('model', awInfo.model); + - name: Install GitHub Copilot CLI + run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.418 + - name: Install awf binary + run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.23.0 + - name: Determine automatic lockdown mode for GitHub MCP Server + id: determine-automatic-lockdown + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + CUSTOM_GITHUB_TOKEN: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + with: + script: | + const determineAutomaticLockdown = require('/opt/gh-aw/actions/determine_automatic_lockdown.cjs'); + await determineAutomaticLockdown(github, context, core); + - name: Download container images + run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.23.0 ghcr.io/github/gh-aw-firewall/api-proxy:0.23.0 ghcr.io/github/gh-aw-firewall/squid:0.23.0 ghcr.io/github/gh-aw-mcpg:v0.1.5 ghcr.io/github/github-mcp-server:v0.31.0 node:lts-alpine + - name: Write Safe Outputs Config + run: | + mkdir -p /opt/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' + {"add_comment":{"max":1,"target":"triggering"},"add_labels":{"allowed":["runtime-fix-needed","sdk-fix-only","needs-investigation"],"max":3,"target":"triggering"},"create_issue":{"max":1},"create_pull_request":{"max":1},"missing_data":{},"missing_tool":{},"noop":{"max":1}} + GH_AW_SAFE_OUTPUTS_CONFIG_EOF + cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' + [ + { + "description": "Create a new GitHub issue for tracking bugs, feature requests, or tasks. Use this for actionable work items that need assignment, labeling, and status tracking. For reports, announcements, or status updates that don't require task tracking, use create_discussion instead. CONSTRAINTS: Maximum 1 issue(s) can be created. Title will be prefixed with \"[copilot-sdk] \". Labels [upstream-from-sdk ai-triaged] will be automatically added. Issues will be created in repository \"github/copilot-agent-runtime\".", + "inputSchema": { + "additionalProperties": false, + "properties": { + "body": { + "description": "Detailed issue description in Markdown. Do NOT repeat the title as a heading since it already appears as the issue's h1. Include context, reproduction steps, or acceptance criteria as appropriate.", + "type": "string" + }, + "labels": { + "description": "Labels to categorize the issue (e.g., 'bug', 'enhancement'). Labels must exist in the repository.", + "items": { + "type": "string" + }, + "type": "array" + }, + "parent": { + "description": "Parent issue number for creating sub-issues. This is the numeric ID from the GitHub URL (e.g., 42 in github.com/owner/repo/issues/42). Can also be a temporary_id (e.g., 'aw_abc123', 'aw_Test123') from a previously created issue in the same workflow run.", + "type": [ + "number", + "string" + ] + }, + "temporary_id": { + "description": "Unique temporary identifier for referencing this issue before it's created. Format: 'aw_' followed by 3 to 8 alphanumeric characters (e.g., 'aw_abc1', 'aw_Test123'). Use '#aw_ID' in body text to reference other issues by their temporary_id; these are replaced with actual issue numbers after creation.", + "pattern": "^aw_[A-Za-z0-9]{3,8}$", + "type": "string" + }, + "title": { + "description": "Concise issue title summarizing the bug, feature, or task. The title appears as the main heading, so keep it brief and descriptive.", + "type": "string" + } + }, + "required": [ + "title", + "body" + ], + "type": "object" + }, + "name": "create_issue" + }, + { + "description": "Add a comment to an existing GitHub issue, pull request, or discussion. Use this to provide feedback, answer questions, or add information to an existing conversation. For creating new items, use create_issue, create_discussion, or create_pull_request instead. IMPORTANT: Comments are subject to validation constraints enforced by the MCP server - maximum 65536 characters for the complete comment (including footer which is added automatically), 10 mentions (@username), and 50 links. Exceeding these limits will result in an immediate error with specific guidance. NOTE: By default, this tool requires discussions:write permission. If your GitHub App lacks Discussions permission, set 'discussions: false' in the workflow's safe-outputs.add-comment configuration to exclude this permission. CONSTRAINTS: Maximum 1 comment(s) can be added. Target: triggering.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "body": { + "description": "The comment text in Markdown format. This is the 'body' field - do not use 'comment_body' or other variations. Provide helpful, relevant information that adds value to the conversation. CONSTRAINTS: The complete comment (your body text + automatically added footer) must not exceed 65536 characters total. Maximum 10 mentions (@username), maximum 50 links (http/https URLs). A footer (~200-500 characters) is automatically appended with workflow attribution, so leave adequate space. If these limits are exceeded, the tool call will fail with a detailed error message indicating which constraint was violated.", + "type": "string" + }, + "item_number": { + "description": "The issue, pull request, or discussion number to comment on. This is the numeric ID from the GitHub URL (e.g., 123 in github.com/owner/repo/issues/123). If omitted, the tool auto-targets the issue, PR, or discussion that triggered this workflow. Auto-targeting only works for issue, pull_request, discussion, and comment event triggers — it does NOT work for schedule, workflow_dispatch, push, or workflow_run triggers. For those trigger types, always provide item_number explicitly, or the comment will be silently discarded.", + "type": "number" + } + }, + "required": [ + "body" + ], + "type": "object" + }, + "name": "add_comment" + }, + { + "description": "Create a new GitHub pull request to propose code changes. Use this after making file edits to submit them for review and merging. The PR will be created from the current branch with your committed changes. For code review comments on an existing PR, use create_pull_request_review_comment instead. CONSTRAINTS: Maximum 1 pull request(s) can be created. Title will be prefixed with \"[copilot-sdk] \". Labels [upstream-from-sdk ai-suggested-fix] will be automatically added. PRs will be created as drafts.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "body": { + "description": "Detailed PR description in Markdown. Include what changes were made, why, testing notes, and any breaking changes. Do NOT repeat the title as a heading.", + "type": "string" + }, + "branch": { + "description": "Source branch name containing the changes. If omitted, uses the current working branch.", + "type": "string" + }, + "draft": { + "description": "Whether to create the PR as a draft. Draft PRs cannot be merged until marked as ready for review. Use mark_pull_request_as_ready_for_review to convert a draft PR. Default: true.", + "type": "boolean" + }, + "labels": { + "description": "Labels to categorize the PR (e.g., 'enhancement', 'bugfix'). Labels must exist in the repository.", + "items": { + "type": "string" + }, + "type": "array" + }, + "title": { + "description": "Concise PR title describing the changes. Follow repository conventions (e.g., conventional commits). The title appears as the main heading.", + "type": "string" + } + }, + "required": [ + "title", + "body" + ], + "type": "object" + }, + "name": "create_pull_request" + }, + { + "description": "Add labels to an existing GitHub issue or pull request for categorization and filtering. Labels must already exist in the repository. For creating new issues with labels, use create_issue with the labels property instead. CONSTRAINTS: Maximum 3 label(s) can be added. Only these labels are allowed: [runtime-fix-needed sdk-fix-only needs-investigation]. Target: triggering.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "item_number": { + "description": "Issue or PR number to add labels to. This is the numeric ID from the GitHub URL (e.g., 456 in github.com/owner/repo/issues/456). If omitted, adds labels to the issue or PR that triggered this workflow. Only works for issue or pull_request event triggers. For schedule, workflow_dispatch, or other triggers, item_number is required — omitting it will silently skip the label operation.", + "type": "number" + }, + "labels": { + "description": "Label names to add (e.g., ['bug', 'priority-high']). Labels must exist in the repository.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "name": "add_labels" + }, + { + "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "reason": { + "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", + "type": "string" + }, + "tool": { + "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", + "type": "string" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "name": "missing_tool" + }, + { + "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "message": { + "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "name": "noop" + }, + { + "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "context": { + "description": "Additional context about the missing data or where it should come from (max 256 characters).", + "type": "string" + }, + "data_type": { + "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", + "type": "string" + }, + "reason": { + "description": "Explanation of why this data is needed to complete the task (max 256 characters).", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "name": "missing_data" + } + ] + GH_AW_SAFE_OUTPUTS_TOOLS_EOF + cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' + { + "add_comment": { + "defaultMax": 1, + "fields": { + "body": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "item_number": { + "issueOrPRNumber": true + }, + "repo": { + "type": "string", + "maxLength": 256 + } + } + }, + "add_labels": { + "defaultMax": 5, + "fields": { + "item_number": { + "issueOrPRNumber": true + }, + "labels": { + "required": true, + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, + "repo": { + "type": "string", + "maxLength": 256 + } + } + }, + "create_issue": { + "defaultMax": 1, + "fields": { + "body": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "labels": { + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, + "parent": { + "issueOrPRNumber": true + }, + "repo": { + "type": "string", + "maxLength": 256 + }, + "temporary_id": { + "type": "string" + }, + "title": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "create_pull_request": { + "defaultMax": 1, + "fields": { + "body": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "branch": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "draft": { + "type": "boolean" + }, + "labels": { + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, + "repo": { + "type": "string", + "maxLength": 256 + }, + "title": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "missing_data": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "context": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "data_type": { + "type": "string", + "sanitize": true, + "maxLength": 128 + }, + "reason": { + "type": "string", + "sanitize": true, + "maxLength": 256 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "tool": { + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "noop": { + "defaultMax": 1, + "fields": { + "message": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + } + } + } + } + GH_AW_SAFE_OUTPUTS_VALIDATION_EOF + - name: Generate Safe Outputs MCP Server Config + id: safe-outputs-config + run: | + # Generate a secure random API key (360 bits of entropy, 40+ chars) + # Mask immediately to prevent timing vulnerabilities + API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${API_KEY}" + + PORT=3001 + + # Set outputs for next steps + { + echo "safe_outputs_api_key=${API_KEY}" + echo "safe_outputs_port=${PORT}" + } >> "$GITHUB_OUTPUT" + + echo "Safe Outputs MCP server will run on port ${PORT}" + + - name: Start Safe Outputs MCP HTTP Server + id: safe-outputs-start + env: + DEBUG: '*' + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + run: | + # Environment variables are set above to prevent template injection + export DEBUG + export GH_AW_SAFE_OUTPUTS_PORT + export GH_AW_SAFE_OUTPUTS_API_KEY + export GH_AW_SAFE_OUTPUTS_TOOLS_PATH + export GH_AW_SAFE_OUTPUTS_CONFIG_PATH + export GH_AW_MCP_LOG_DIR + + bash /opt/gh-aw/actions/start_safe_outputs_server.sh + + - name: Start MCP Gateway + id: start-mcp-gateway + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} + GITHUB_MCP_LOCKDOWN: ${{ steps.determine-automatic-lockdown.outputs.lockdown == 'true' && '1' || '0' }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + run: | + set -eo pipefail + mkdir -p /tmp/gh-aw/mcp-config + + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="80" + export MCP_GATEWAY_DOMAIN="host.docker.internal" + MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_API_KEY}" + export MCP_GATEWAY_API_KEY + export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" + mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export DEBUG="*" + + export GH_AW_ENGINE="copilot" + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.5' + + mkdir -p /home/runner/.copilot + cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway.sh + { + "mcpServers": { + "github": { + "type": "stdio", + "container": "ghcr.io/github/github-mcp-server:v0.31.0", + "env": { + "GITHUB_LOCKDOWN_MODE": "$GITHUB_MCP_LOCKDOWN", + "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", + "GITHUB_READ_ONLY": "1", + "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" + } + }, + "safeoutputs": { + "type": "http", + "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", + "headers": { + "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + } + } + }, + "gateway": { + "port": $MCP_GATEWAY_PORT, + "domain": "${MCP_GATEWAY_DOMAIN}", + "apiKey": "${MCP_GATEWAY_API_KEY}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" + } + } + GH_AW_MCP_CONFIG_EOF + - name: Generate workflow overview + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); + await generateWorkflowOverview(core); + - name: Download prompt artifact + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + name: prompt + path: /tmp/gh-aw/aw-prompts + - name: Clean git credentials + run: bash /opt/gh-aw/actions/clean_git_credentials.sh + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + # --allow-tool github + # --allow-tool safeoutputs + # --allow-tool shell(cat) + # --allow-tool shell(cat:*) + # --allow-tool shell(date) + # --allow-tool shell(echo) + # --allow-tool shell(find:*) + # --allow-tool shell(git add:*) + # --allow-tool shell(git branch:*) + # --allow-tool shell(git checkout:*) + # --allow-tool shell(git commit:*) + # --allow-tool shell(git merge:*) + # --allow-tool shell(git rm:*) + # --allow-tool shell(git status) + # --allow-tool shell(git switch:*) + # --allow-tool shell(grep) + # --allow-tool shell(grep:*) + # --allow-tool shell(head) + # --allow-tool shell(head:*) + # --allow-tool shell(ls) + # --allow-tool shell(ls:*) + # --allow-tool shell(pwd) + # --allow-tool shell(sort) + # --allow-tool shell(tail) + # --allow-tool shell(tail:*) + # --allow-tool shell(uniq) + # --allow-tool shell(wc) + # --allow-tool shell(wc:*) + # --allow-tool shell(yq) + # --allow-tool write + timeout-minutes: 20 + run: | + set -o pipefail + # shellcheck disable=SC1003 + sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.23.0 --skip-pull --enable-api-proxy \ + -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(cat:*)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(find:*)'\'' --allow-tool '\''shell(git add:*)'\'' --allow-tool '\''shell(git branch:*)'\'' --allow-tool '\''shell(git checkout:*)'\'' --allow-tool '\''shell(git commit:*)'\'' --allow-tool '\''shell(git merge:*)'\'' --allow-tool '\''shell(git rm:*)'\'' --allow-tool '\''shell(git status)'\'' --allow-tool '\''shell(git switch:*)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(grep:*)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(head:*)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(ls:*)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(tail:*)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(wc:*)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_AGENT_COPILOT:+ --model "$GH_AW_MODEL_AGENT_COPILOT"}' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json + GH_AW_MODEL_AGENT_COPILOT: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_WORKSPACE: ${{ github.workspace }} + XDG_CONFIG_HOME: /home/runner + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Copy Copilot session state files to logs + if: always() + continue-on-error: true + run: | + # Copy Copilot session state files to logs folder for artifact collection + # This ensures they are in /tmp/gh-aw/ where secret redaction can scan them + SESSION_STATE_DIR="$HOME/.copilot/session-state" + LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" + + if [ -d "$SESSION_STATE_DIR" ]; then + echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" + mkdir -p "$LOGS_DIR" + cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true + echo "Session state files copied successfully" + else + echo "No session-state directory found at $SESSION_STATE_DIR" + fi + - name: Stop MCP Gateway + if: always() + continue-on-error: true + env: + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} + run: | + bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" + - name: Redact secrets in logs + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + await main(); + env: + GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,RUNTIME_TRIAGE_TOKEN' + SECRET_COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_RUNTIME_TRIAGE_TOKEN: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + - name: Upload Safe Outputs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: safe-output + path: ${{ env.GH_AW_SAFE_OUTPUTS }} + if-no-files-found: warn + - name: Ingest agent output + id: collect_output + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GH_AW_ALLOWED_GITHUB_REFS: "repo,github/copilot-agent-runtime" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + await main(); + - name: Upload sanitized agent output + if: always() && env.GH_AW_AGENT_OUTPUT + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: agent-output + path: ${{ env.GH_AW_AGENT_OUTPUT }} + if-no-files-found: warn + - name: Upload engine output files + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: agent_outputs + path: | + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log + if-no-files-found: ignore + - name: Parse agent logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + await main(); + - name: Parse MCP Gateway logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Print firewall logs + if: always() + continue-on-error: true + env: + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs + run: | + # Fix permissions on firewall logs so they can be uploaded as artifacts + # AWF runs with sudo, creating files owned by root + sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || true + # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) + if command -v awf &> /dev/null; then + awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" + else + echo 'AWF binary not installed, skipping firewall log summary' + fi + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: agent-artifacts + path: | + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/mcp-logs/ + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/agent/ + /tmp/gh-aw/aw-*.patch + if-no-files-found: ignore + # --- Threat Detection (inline) --- + - name: Check if detection needed + id: detection_guard + if: always() + env: + OUTPUT_TYPES: ${{ steps.collect_output.outputs.output_types }} + HAS_PATCH: ${{ steps.collect_output.outputs.has_patch }} + run: | + if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then + echo "run_detection=true" >> "$GITHUB_OUTPUT" + echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" + else + echo "run_detection=false" >> "$GITHUB_OUTPUT" + echo "Detection skipped: no agent outputs or patches to analyze" + fi + - name: Clear MCP configuration for detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + rm -f /tmp/gh-aw/mcp-config/mcp-servers.json + rm -f /home/runner/.copilot/mcp-config.json + rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" + - name: Prepare threat detection files + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection/aw-prompts + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true + for f in /tmp/gh-aw/aw-*.patch; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + echo "Prepared threat detection files:" + ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true + - name: Setup threat detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + WORKFLOW_NAME: "SDK Runtime Triage" + WORKFLOW_DESCRIPTION: "Analyzes copilot-sdk issues to determine if a fix is needed in copilot-agent-runtime, then opens a linked issue and suggested-fix PR there" + HAS_PATCH: ${{ steps.collect_output.outputs.has_patch }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + await main(); + - name: Ensure threat-detection directory and log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Execute GitHub Copilot CLI + if: always() && steps.detection_guard.outputs.run_detection == 'true' + id: detection_agentic_execution + # Copilot CLI tool arguments (sorted): + # --allow-tool shell(cat) + # --allow-tool shell(grep) + # --allow-tool shell(head) + # --allow-tool shell(jq) + # --allow-tool shell(ls) + # --allow-tool shell(tail) + # --allow-tool shell(wc) + timeout-minutes: 20 + run: | + set -o pipefail + # shellcheck disable=SC1003 + sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,github.com,host.docker.internal,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com" --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.23.0 --skip-pull --enable-api-proxy \ + -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(jq)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(wc)'\'' --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_DETECTION_COPILOT:+ --model "$GH_AW_MODEL_DETECTION_COPILOT"}' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_MODEL_DETECTION_COPILOT: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_WORKSPACE: ${{ github.workspace }} + XDG_CONFIG_HOME: /home/runner + - name: Parse threat detection results + id: parse_detection_results + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + - name: Upload threat detection log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: threat-detection.log + path: /tmp/gh-aw/threat-detection/detection.log + if-no-files-found: ignore + - name: Set detection conclusion + id: detection_conclusion + if: always() + env: + RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} + DETECTION_SUCCESS: ${{ steps.parse_detection_results.outputs.success }} + run: | + if [[ "$RUN_DETECTION" != "true" ]]; then + echo "conclusion=skipped" >> "$GITHUB_OUTPUT" + echo "success=true" >> "$GITHUB_OUTPUT" + echo "Detection was not needed, marking as skipped" + elif [[ "$DETECTION_SUCCESS" == "true" ]]; then + echo "conclusion=success" >> "$GITHUB_OUTPUT" + echo "success=true" >> "$GITHUB_OUTPUT" + echo "Detection passed successfully" + else + echo "conclusion=failure" >> "$GITHUB_OUTPUT" + echo "success=false" >> "$GITHUB_OUTPUT" + echo "Detection found issues" + fi + + conclusion: + needs: + - activation + - agent + - safe_outputs + if: (always()) && (needs.agent.result != 'skipped') + runs-on: ubuntu-slim + permissions: + contents: write + discussions: write + issues: write + pull-requests: write + outputs: + noop_message: ${{ steps.noop.outputs.noop_message }} + tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} + total_count: ${{ steps.missing_tool.outputs.total_count }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@a7d371cc7e68f270ded0592942424548e05bf1c2 # v0.50.5 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Process No-Op Messages + id: noop + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: "1" + GH_AW_WORKFLOW_NAME: "SDK Runtime Triage" + with: + github-token: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/noop.cjs'); + await main(); + - name: Record Missing Tool + id: missing_tool + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "SDK Runtime Triage" + with: + github-token: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + await main(); + - name: Handle Agent Failure + id: handle_agent_failure + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "SDK Runtime Triage" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_WORKFLOW_ID: "cross-repo-issue-analysis" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + GH_AW_CODE_PUSH_FAILURE_ERRORS: ${{ needs.safe_outputs.outputs.code_push_failure_errors }} + GH_AW_CODE_PUSH_FAILURE_COUNT: ${{ needs.safe_outputs.outputs.code_push_failure_count }} + GH_AW_GROUP_REPORTS: "false" + with: + github-token: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + await main(); + - name: Handle No-Op Message + id: handle_noop_message + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "SDK Runtime Triage" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" + with: + github-token: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + await main(); + - name: Handle Create Pull Request Error + id: handle_create_pr_error + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "SDK Runtime Triage" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + with: + github-token: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_create_pr_error.cjs'); + await main(); + + pre_activation: + runs-on: ubuntu-slim + outputs: + activated: ${{ steps.check_membership.outputs.is_team_member == 'true' }} + matched_command: '' + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@a7d371cc7e68f270ded0592942424548e05bf1c2 # v0.50.5 + with: + destination: /opt/gh-aw/actions + - name: Check team membership for workflow + id: check_membership + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_REQUIRED_ROLES: admin,maintainer,write + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/check_membership.cjs'); + await main(); + + safe_outputs: + needs: + - activation + - agent + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.agent.outputs.detection_success == 'true') + runs-on: ubuntu-slim + permissions: + contents: write + discussions: write + issues: write + pull-requests: write + timeout-minutes: 15 + env: + GH_AW_ENGINE_ID: "copilot" + GH_AW_WORKFLOW_ID: "cross-repo-issue-analysis" + GH_AW_WORKFLOW_NAME: "SDK Runtime Triage" + outputs: + code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} + code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} + create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} + create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} + process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@a7d371cc7e68f270ded0592942424548e05bf1c2 # v0.50.5 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Download patch artifact + continue-on-error: true + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + name: agent-artifacts + path: /tmp/gh-aw/ + - name: Checkout repository + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (contains(needs.agent.outputs.output_types, 'create_pull_request')) + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: github/copilot-agent-runtime + ref: ${{ github.base_ref || github.ref_name }} + token: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + persist-credentials: false + fetch-depth: 1 + - name: Configure Git credentials + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (contains(needs.agent.outputs.output_types, 'create_pull_request')) + env: + REPO_NAME: "github/copilot-agent-runtime" + SERVER_URL: ${{ github.server_url }} + GIT_TOKEN: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${GIT_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":1,\"target\":\"triggering\"},\"add_labels\":{\"allowed\":[\"runtime-fix-needed\",\"sdk-fix-only\",\"needs-investigation\"],\"max\":3,\"target\":\"triggering\"},\"create_issue\":{\"labels\":[\"upstream-from-sdk\",\"ai-triaged\"],\"max\":1,\"target-repo\":\"github/copilot-agent-runtime\",\"title_prefix\":\"[copilot-sdk] \"},\"create_pull_request\":{\"base_branch\":\"${{ github.base_ref || github.ref_name }}\",\"draft\":true,\"labels\":[\"upstream-from-sdk\",\"ai-suggested-fix\"],\"max\":1,\"max_patch_size\":1024,\"target-repo\":\"github/copilot-agent-runtime\",\"title_prefix\":\"[copilot-sdk] \"},\"missing_data\":{},\"missing_tool\":{}}" + GH_AW_CI_TRIGGER_TOKEN: ${{ secrets.GH_AW_CI_TRIGGER_TOKEN }} + with: + github-token: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + await main(); + - name: Upload safe output items manifest + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: safe-output-items + path: /tmp/safe-output-items.jsonl + if-no-files-found: warn + diff --git a/.github/workflows/cross-repo-issue-analysis.md b/.github/workflows/cross-repo-issue-analysis.md new file mode 100644 index 000000000..76807bbcb --- /dev/null +++ b/.github/workflows/cross-repo-issue-analysis.md @@ -0,0 +1,126 @@ +--- +description: Analyzes copilot-sdk issues to determine if a fix is needed in copilot-agent-runtime, then opens a linked issue and suggested-fix PR there +on: + issues: + types: [opened] + workflow_dispatch: + inputs: + issue_number: + description: "Issue number to analyze" + required: true + type: string +permissions: + contents: read + issues: read + pull-requests: read +steps: + - name: Clone copilot-agent-runtime + run: git clone --depth 1 https://x-access-token:${{ secrets.RUNTIME_TRIAGE_TOKEN }}@github.com/github/copilot-agent-runtime.git ${{ github.workspace }}/copilot-agent-runtime +tools: + github: + toolsets: [default] + github-token: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + edit: + bash: + - "grep:*" + - "find:*" + - "cat:*" + - "head:*" + - "tail:*" + - "wc:*" + - "ls:*" +safe-outputs: + github-token: ${{ secrets.RUNTIME_TRIAGE_TOKEN }} + allowed-github-references: ["repo", "github/copilot-agent-runtime"] + add-comment: + max: 1 + target: triggering + add-labels: + allowed: [runtime-fix-needed, sdk-fix-only, needs-investigation] + max: 3 + target: triggering + create-issue: + title-prefix: "[copilot-sdk] " + labels: [upstream-from-sdk, ai-triaged] + target-repo: "github/copilot-agent-runtime" + max: 1 + create-pull-request: + title-prefix: "[copilot-sdk] " + labels: [upstream-from-sdk, ai-suggested-fix] + draft: true + target-repo: "github/copilot-agent-runtime" + +timeout-minutes: 20 +--- + +# SDK Runtime Triage + +You are an expert agent that analyzes issues filed in the **copilot-sdk** repository to determine whether the root cause and fix live in this repo or in the **copilot-agent-runtime** repo (`github/copilot-agent-runtime`). + +## Context + +- Repository: ${{ github.repository }} +- Issue number: ${{ github.event.issue.number || inputs.issue_number }} +- Issue title: ${{ github.event.issue.title }} + +The **copilot-sdk** repo is a multi-language SDK (Node/TS, Python, Go, .NET) that communicates with the Copilot CLI via JSON-RPC. The **copilot-agent-runtime** repo contains the CLI/server that the SDK talks to. Many issues filed against the SDK are actually caused by behavior in the runtime. + +## Your Task + +### Step 1: Understand the Issue + +Use GitHub tools to fetch the full issue body, comments, and any linked references for issue `${{ github.event.issue.number || inputs.issue_number }}` in `${{ github.repository }}`. + +### Step 2: Analyze Against copilot-sdk + +Search the copilot-sdk codebase on disk to understand whether the reported problem could originate here. The repo is checked out at the default working directory. + +- Use bash tools (`grep`, `find`, `cat`) to search the relevant SDK language implementation (`nodejs/src/`, `python/copilot/`, `go/`, `dotnet/src/`) +- Look at the JSON-RPC client layer, session management, event handling, and tool definitions +- Check if the issue relates to SDK-side logic (type generation, streaming, event parsing, client options, etc.) + +### Step 3: Investigate copilot-agent-runtime + +If the issue does NOT appear to be caused by SDK code, or you suspect the runtime is involved, investigate the **copilot-agent-runtime** repo. It has been cloned to `./copilot-agent-runtime/` in the current working directory. + +- Use bash tools (`grep`, `find`, `cat`) to search the runtime codebase at `./copilot-agent-runtime/` +- Look at the server-side JSON-RPC handling, session management, tool execution, and response generation +- Focus on the areas that correspond to the reported issue (e.g., if the issue is about streaming, look at the runtime's streaming implementation) + +Common areas where runtime fixes are needed: +- JSON-RPC protocol handling and response formatting +- Session lifecycle (creation, persistence, compaction, destruction) +- Tool execution and permission handling +- Model/API interaction (prompt construction, response parsing) +- Streaming event generation (deltas, completions) +- Error handling and error response formatting + +### Step 4: Make Your Determination + +Classify the issue into one of these categories: + +1. **SDK-fix-only**: The bug/feature is entirely in the SDK code. Label the issue `sdk-fix-only` and comment with your analysis. + +2. **Runtime-fix-needed**: The root cause is in copilot-agent-runtime. Do ALL of the following: + - Label the original issue `runtime-fix-needed` + - Create an issue in `github/copilot-agent-runtime` that: + - Clearly describes the problem and root cause + - References the original SDK issue (e.g., `github/copilot-sdk#123`) + - Includes the specific files and code paths involved + - Suggests a fix approach + - Create a draft PR in `github/copilot-agent-runtime` with a suggested fix: + - Make the minimal, targeted code changes needed + - Include a clear PR description linking back to both issues + - If you're uncertain about the fix, still create the PR as a starting point for discussion + - Comment on the original SDK issue summarizing your findings and linking to the new runtime issue and PR + +3. **Needs-investigation**: You cannot confidently determine the root cause. Label the issue `needs-investigation` and comment explaining what you found and what needs further human review. + +## Guidelines + +1. **Be thorough but focused**: Read enough code to be confident in your analysis, but don't read every file in both repos +2. **Err on the side of creating the runtime issue**: If there's a reasonable chance the fix is in the runtime, create the issue. False positives are better than missed upstream bugs. +3. **Make actionable PRs**: Even if the fix isn't perfect, a draft PR with a concrete starting point is more useful than just an issue description +4. **Link everything**: Always cross-reference between the SDK issue, runtime issue, and runtime PR so maintainers can follow the trail +5. **Be specific**: When describing the root cause, point to specific files, functions, and line numbers in both repos +6. **Don't duplicate**: Before creating a runtime issue, search existing open issues in `github/copilot-agent-runtime` to avoid duplicates. If a related issue exists, reference it instead of creating a new one. diff --git a/dotnet/test/Harness/E2ETestContext.cs b/dotnet/test/Harness/E2ETestContext.cs index 00fc32075..62acd580f 100644 --- a/dotnet/test/Harness/E2ETestContext.cs +++ b/dotnet/test/Harness/E2ETestContext.cs @@ -94,13 +94,13 @@ public IReadOnlyDictionary GetEnvironment() Cwd = WorkDir, CliPath = GetCliPath(_repoRoot), Environment = GetEnvironment(), - GitHubToken = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("CI")) ? "fake-token-for-e2e-tests" : null, + GitHubToken = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS")) ? "fake-token-for-e2e-tests" : null, }); public async ValueTask DisposeAsync() { // Skip writing snapshots in CI to avoid corrupting them on test failures - var isCI = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("CI")); + var isCI = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS")); await _proxy.StopAsync(skipWritingCache: isCI); try { if (Directory.Exists(HomeDir)) Directory.Delete(HomeDir, true); } catch { } diff --git a/dotnet/test/SessionTests.cs b/dotnet/test/SessionTests.cs index eac00b06e..c202bc00b 100644 --- a/dotnet/test/SessionTests.cs +++ b/dotnet/test/SessionTests.cs @@ -167,6 +167,11 @@ public async Task Should_Resume_A_Session_Using_The_Same_Client() var answer2 = await TestHelper.GetFinalAssistantMessageAsync(session2); Assert.NotNull(answer2); Assert.Contains("2", answer2!.Data.Content ?? string.Empty); + + // Can continue the conversation statefully + var answer3 = await session2.SendAndWaitAsync(new MessageOptions { Prompt = "Now if you double that, what do you get?" }); + Assert.NotNull(answer3); + Assert.Contains("4", answer3!.Data.Content ?? string.Empty); } [Fact] @@ -187,6 +192,11 @@ public async Task Should_Resume_A_Session_Using_A_New_Client() var messages = await session2.GetMessagesAsync(); Assert.Contains(messages, m => m is UserMessageEvent); Assert.Contains(messages, m => m is SessionResumeEvent); + + // Can continue the conversation statefully + var answer2 = await session2.SendAndWaitAsync(new MessageOptions { Prompt = "Now if you double that, what do you get?" }); + Assert.NotNull(answer2); + Assert.Contains("4", answer2!.Data.Content ?? string.Empty); } [Fact] @@ -231,68 +241,6 @@ await session.SendAsync(new MessageOptions Assert.Contains("4", answer!.Data.Content ?? string.Empty); } - // TODO: This test requires the session-events.schema.json to include assistant.message_delta. - // The CLI v0.0.376 emits delta events at runtime, but the schema hasn't been updated yet. - // Once the schema is updated and types are regenerated, this test can be enabled. - [Fact(Skip = "Requires schema update for AssistantMessageDeltaEvent type")] - public async Task Should_Receive_Streaming_Delta_Events_When_Streaming_Is_Enabled() - { - var session = await CreateSessionAsync(new SessionConfig { Streaming = true }); - - var deltaContents = new List(); - var doneEvent = new TaskCompletionSource(); - - session.On(evt => - { - switch (evt) - { - // TODO: Uncomment once AssistantMessageDeltaEvent is generated - // case AssistantMessageDeltaEvent delta: - // if (!string.IsNullOrEmpty(delta.Data.DeltaContent)) - // deltaContents.Add(delta.Data.DeltaContent); - // break; - case SessionIdleEvent: - doneEvent.TrySetResult(true); - break; - } - }); - - await session.SendAsync(new MessageOptions { Prompt = "What is 2+2?" }); - - // Wait for completion - var completed = await Task.WhenAny(doneEvent.Task, Task.Delay(TimeSpan.FromSeconds(60))); - Assert.Equal(doneEvent.Task, completed); - - // Should have received delta events - Assert.NotEmpty(deltaContents); - - // Get the final message to compare - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); - Assert.NotNull(assistantMessage); - - // Accumulated deltas should equal the final message - var accumulated = string.Join("", deltaContents); - Assert.Equal(assistantMessage!.Data.Content, accumulated); - - // Final message should contain the answer - Assert.Contains("4", assistantMessage.Data.Content ?? string.Empty); - } - - [Fact] - public async Task Should_Pass_Streaming_Option_To_Session_Creation() - { - // Verify that the streaming option is accepted without errors - var session = await CreateSessionAsync(new SessionConfig { Streaming = true }); - - Assert.Matches(@"^[a-f0-9-]+$", session.SessionId); - - // Session should still work normally - await session.SendAsync(new MessageOptions { Prompt = "What is 1+1?" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); - Assert.NotNull(assistantMessage); - Assert.Contains("2", assistantMessage!.Data.Content); - } - [Fact] public async Task Should_Receive_Session_Events() { diff --git a/dotnet/test/StreamingFidelityTests.cs b/dotnet/test/StreamingFidelityTests.cs new file mode 100644 index 000000000..c38cb1545 --- /dev/null +++ b/dotnet/test/StreamingFidelityTests.cs @@ -0,0 +1,99 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +using GitHub.Copilot.SDK.Test.Harness; +using Xunit; +using Xunit.Abstractions; + +namespace GitHub.Copilot.SDK.Test; + +public class StreamingFidelityTests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "streaming_fidelity", output) +{ + [Fact] + public async Task Should_Produce_Delta_Events_When_Streaming_Is_Enabled() + { + var session = await CreateSessionAsync(new SessionConfig { Streaming = true }); + + var events = new List(); + session.On(evt => events.Add(evt)); + + await session.SendAndWaitAsync(new MessageOptions { Prompt = "Count from 1 to 5, separated by commas." }); + + var types = events.Select(e => e.Type).ToList(); + + // Should have streaming deltas before the final message + var deltaEvents = events.OfType().ToList(); + Assert.NotEmpty(deltaEvents); + + // Deltas should have content + foreach (var delta in deltaEvents) + { + Assert.False(string.IsNullOrEmpty(delta.Data.DeltaContent)); + } + + // Should still have a final assistant.message + Assert.Contains("assistant.message", types); + + // Deltas should come before the final message + var firstDeltaIdx = types.IndexOf("assistant.message_delta"); + var lastAssistantIdx = types.LastIndexOf("assistant.message"); + Assert.True(firstDeltaIdx < lastAssistantIdx); + + await session.DisposeAsync(); + } + + [Fact] + public async Task Should_Not_Produce_Deltas_When_Streaming_Is_Disabled() + { + var session = await CreateSessionAsync(new SessionConfig { Streaming = false }); + + var events = new List(); + session.On(evt => events.Add(evt)); + + await session.SendAndWaitAsync(new MessageOptions { Prompt = "Say 'hello world'." }); + + var deltaEvents = events.OfType().ToList(); + + // No deltas when streaming is off + Assert.Empty(deltaEvents); + + // But should still have a final assistant.message + var assistantEvents = events.OfType().ToList(); + Assert.NotEmpty(assistantEvents); + + await session.DisposeAsync(); + } + + [Fact] + public async Task Should_Produce_Deltas_After_Session_Resume() + { + var session = await CreateSessionAsync(new SessionConfig { Streaming = false }); + await session.SendAndWaitAsync(new MessageOptions { Prompt = "What is 3 + 6?" }); + await session.DisposeAsync(); + + // Resume using a new client + using var newClient = Ctx.CreateClient(); + var session2 = await newClient.ResumeSessionAsync(session.SessionId, + new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Streaming = true }); + + var events = new List(); + session2.On(evt => events.Add(evt)); + + var answer = await session2.SendAndWaitAsync(new MessageOptions { Prompt = "Now if you double that, what do you get?" }); + Assert.NotNull(answer); + Assert.Contains("18", answer!.Data.Content ?? string.Empty); + + // Should have streaming deltas before the final message + var deltaEvents = events.OfType().ToList(); + Assert.NotEmpty(deltaEvents); + + // Deltas should have content + foreach (var delta in deltaEvents) + { + Assert.False(string.IsNullOrEmpty(delta.Data.DeltaContent)); + } + + await session2.DisposeAsync(); + } +} diff --git a/go/internal/e2e/session_test.go b/go/internal/e2e/session_test.go index f04307c2d..0c50ba8d9 100644 --- a/go/internal/e2e/session_test.go +++ b/go/internal/e2e/session_test.go @@ -368,6 +368,15 @@ func TestSession(t *testing.T) { if answer2.Data.Content == nil || !strings.Contains(*answer2.Data.Content, "2") { t.Errorf("Expected resumed session answer to contain '2', got %v", answer2.Data.Content) } + + // Can continue the conversation statefully + answer3, err := session2.SendAndWait(t.Context(), copilot.MessageOptions{Prompt: "Now if you double that, what do you get?"}) + if err != nil { + t.Fatalf("Failed to send follow-up message: %v", err) + } + if answer3 == nil || answer3.Data.Content == nil || !strings.Contains(*answer3.Data.Content, "4") { + t.Errorf("Expected follow-up answer to contain '4', got %v", answer3) + } }) t.Run("should resume a session using a new client", func(t *testing.T) { @@ -432,6 +441,15 @@ func TestSession(t *testing.T) { if !hasSessionResume { t.Error("Expected messages to contain 'session.resume'") } + + // Can continue the conversation statefully + answer3, err := session2.SendAndWait(t.Context(), copilot.MessageOptions{Prompt: "Now if you double that, what do you get?"}) + if err != nil { + t.Fatalf("Failed to send follow-up message: %v", err) + } + if answer3 == nil || answer3.Data.Content == nil || !strings.Contains(*answer3.Data.Content, "4") { + t.Errorf("Expected follow-up answer to contain '4', got %v", answer3) + } }) t.Run("should throw error when resuming non-existent session", func(t *testing.T) { @@ -565,99 +583,6 @@ func TestSession(t *testing.T) { } }) - t.Run("should receive streaming delta events when streaming is enabled", func(t *testing.T) { - ctx.ConfigureForTest(t) - - session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{ - OnPermissionRequest: copilot.PermissionHandler.ApproveAll, - Streaming: true, - }) - if err != nil { - t.Fatalf("Failed to create session with streaming: %v", err) - } - - var deltaContents []string - done := make(chan bool) - - session.On(func(event copilot.SessionEvent) { - switch event.Type { - case "assistant.message_delta": - if event.Data.DeltaContent != nil { - deltaContents = append(deltaContents, *event.Data.DeltaContent) - } - case "session.idle": - close(done) - } - }) - - _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What is 2+2?"}) - if err != nil { - t.Fatalf("Failed to send message: %v", err) - } - - // Wait for completion - select { - case <-done: - case <-time.After(60 * time.Second): - t.Fatal("Timed out waiting for session.idle") - } - - // Should have received delta events - if len(deltaContents) == 0 { - t.Error("Expected to receive delta events, got none") - } - - // Get the final message to compare - assistantMessage, err := testharness.GetFinalAssistantMessage(t.Context(), session) - if err != nil { - t.Fatalf("Failed to get assistant message: %v", err) - } - - // Accumulated deltas should equal the final message - accumulated := strings.Join(deltaContents, "") - if assistantMessage.Data.Content != nil && accumulated != *assistantMessage.Data.Content { - t.Errorf("Accumulated deltas don't match final message.\nAccumulated: %q\nFinal: %q", accumulated, *assistantMessage.Data.Content) - } - - // Final message should contain the answer - if assistantMessage.Data.Content == nil || !strings.Contains(*assistantMessage.Data.Content, "4") { - t.Errorf("Expected assistant message to contain '4', got %v", assistantMessage.Data.Content) - } - }) - - t.Run("should pass streaming option to session creation", func(t *testing.T) { - ctx.ConfigureForTest(t) - - // Verify that the streaming option is accepted without errors - session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{ - OnPermissionRequest: copilot.PermissionHandler.ApproveAll, - Streaming: true, - }) - if err != nil { - t.Fatalf("Failed to create session with streaming: %v", err) - } - - matched, _ := regexp.MatchString(`^[a-f0-9-]+$`, session.SessionID) - if !matched { - t.Errorf("Expected session ID to match UUID pattern, got %q", session.SessionID) - } - - // Session should still work normally - _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What is 1+1?"}) - if err != nil { - t.Fatalf("Failed to send message: %v", err) - } - - assistantMessage, err := testharness.GetFinalAssistantMessage(t.Context(), session) - if err != nil { - t.Fatalf("Failed to get assistant message: %v", err) - } - - if assistantMessage.Data.Content == nil || !strings.Contains(*assistantMessage.Data.Content, "2") { - t.Errorf("Expected assistant message to contain '2', got %v", assistantMessage.Data.Content) - } - }) - t.Run("should receive session events", func(t *testing.T) { ctx.ConfigureForTest(t) diff --git a/go/internal/e2e/streaming_fidelity_test.go b/go/internal/e2e/streaming_fidelity_test.go new file mode 100644 index 000000000..ef76c3d8b --- /dev/null +++ b/go/internal/e2e/streaming_fidelity_test.go @@ -0,0 +1,186 @@ +package e2e + +import ( + "strings" + "testing" + + copilot "github.com/github/copilot-sdk/go" + "github.com/github/copilot-sdk/go/internal/e2e/testharness" +) + +func TestStreamingFidelity(t *testing.T) { + ctx := testharness.NewTestContext(t) + client := ctx.NewClient() + t.Cleanup(func() { client.ForceStop() }) + + t.Run("should produce delta events when streaming is enabled", func(t *testing.T) { + ctx.ConfigureForTest(t) + + session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{ + OnPermissionRequest: copilot.PermissionHandler.ApproveAll, + Streaming: true, + }) + if err != nil { + t.Fatalf("Failed to create session with streaming: %v", err) + } + + var events []copilot.SessionEvent + session.On(func(event copilot.SessionEvent) { + events = append(events, event) + }) + + _, err = session.SendAndWait(t.Context(), copilot.MessageOptions{Prompt: "Count from 1 to 5, separated by commas."}) + if err != nil { + t.Fatalf("Failed to send message: %v", err) + } + + // Should have streaming deltas before the final message + var deltaEvents []copilot.SessionEvent + for _, e := range events { + if e.Type == "assistant.message_delta" { + deltaEvents = append(deltaEvents, e) + } + } + if len(deltaEvents) < 1 { + t.Error("Expected at least 1 delta event") + } + + // Deltas should have content + for _, delta := range deltaEvents { + if delta.Data.DeltaContent == nil { + t.Error("Expected delta to have content") + } + } + + // Should still have a final assistant.message + hasAssistantMessage := false + for _, e := range events { + if e.Type == "assistant.message" { + hasAssistantMessage = true + break + } + } + if !hasAssistantMessage { + t.Error("Expected a final assistant.message event") + } + + // Deltas should come before the final message + firstDeltaIdx := -1 + lastAssistantIdx := -1 + for i, e := range events { + if e.Type == "assistant.message_delta" && firstDeltaIdx == -1 { + firstDeltaIdx = i + } + if e.Type == "assistant.message" { + lastAssistantIdx = i + } + } + if firstDeltaIdx >= lastAssistantIdx { + t.Errorf("Expected deltas before final message, got delta at %d, message at %d", firstDeltaIdx, lastAssistantIdx) + } + }) + + t.Run("should not produce deltas when streaming is disabled", func(t *testing.T) { + ctx.ConfigureForTest(t) + + session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{ + OnPermissionRequest: copilot.PermissionHandler.ApproveAll, + Streaming: false, + }) + if err != nil { + t.Fatalf("Failed to create session: %v", err) + } + + var events []copilot.SessionEvent + session.On(func(event copilot.SessionEvent) { + events = append(events, event) + }) + + _, err = session.SendAndWait(t.Context(), copilot.MessageOptions{Prompt: "Say 'hello world'."}) + if err != nil { + t.Fatalf("Failed to send message: %v", err) + } + + // No deltas when streaming is off + var deltaEvents []copilot.SessionEvent + for _, e := range events { + if e.Type == "assistant.message_delta" { + deltaEvents = append(deltaEvents, e) + } + } + if len(deltaEvents) != 0 { + t.Errorf("Expected no delta events, got %d", len(deltaEvents)) + } + + // But should still have a final assistant.message + var assistantEvents []copilot.SessionEvent + for _, e := range events { + if e.Type == "assistant.message" { + assistantEvents = append(assistantEvents, e) + } + } + if len(assistantEvents) < 1 { + t.Error("Expected at least 1 assistant.message event") + } + }) + + t.Run("should produce deltas after session resume", func(t *testing.T) { + ctx.ConfigureForTest(t) + + session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{ + OnPermissionRequest: copilot.PermissionHandler.ApproveAll, + Streaming: false, + }) + if err != nil { + t.Fatalf("Failed to create session: %v", err) + } + + _, err = session.SendAndWait(t.Context(), copilot.MessageOptions{Prompt: "What is 3 + 6?"}) + if err != nil { + t.Fatalf("Failed to send message: %v", err) + } + + // Resume using a new client + newClient := ctx.NewClient() + defer newClient.ForceStop() + + session2, err := newClient.ResumeSession(t.Context(), session.SessionID, &copilot.ResumeSessionConfig{ + OnPermissionRequest: copilot.PermissionHandler.ApproveAll, + Streaming: true, + }) + if err != nil { + t.Fatalf("Failed to resume session: %v", err) + } + + var events []copilot.SessionEvent + session2.On(func(event copilot.SessionEvent) { + events = append(events, event) + }) + + answer, err := session2.SendAndWait(t.Context(), copilot.MessageOptions{Prompt: "Now if you double that, what do you get?"}) + if err != nil { + t.Fatalf("Failed to send follow-up message: %v", err) + } + if answer == nil || answer.Data.Content == nil || !strings.Contains(*answer.Data.Content, "18") { + t.Errorf("Expected answer to contain '18', got %v", answer) + } + + // Should have streaming deltas before the final message + var deltaEvents []copilot.SessionEvent + for _, e := range events { + if e.Type == "assistant.message_delta" { + deltaEvents = append(deltaEvents, e) + } + } + if len(deltaEvents) < 1 { + t.Error("Expected at least 1 delta event") + } + + // Deltas should have content + for _, delta := range deltaEvents { + if delta.Data.DeltaContent == nil { + t.Error("Expected delta to have content") + } + } + }) +} diff --git a/go/internal/e2e/testharness/context.go b/go/internal/e2e/testharness/context.go index cefb87b58..b9edab1e5 100644 --- a/go/internal/e2e/testharness/context.go +++ b/go/internal/e2e/testharness/context.go @@ -166,7 +166,7 @@ func (c *TestContext) NewClient() *copilot.Client { } // Use fake token in CI to allow cached responses without real auth - if os.Getenv("CI") == "true" { + if os.Getenv("GITHUB_ACTIONS") == "true" { options.GitHubToken = "fake-token-for-e2e-tests" } diff --git a/nodejs/test/e2e/harness/sdkTestContext.ts b/nodejs/test/e2e/harness/sdkTestContext.ts index 4986d1299..a5cf2ec57 100644 --- a/nodejs/test/e2e/harness/sdkTestContext.ts +++ b/nodejs/test/e2e/harness/sdkTestContext.ts @@ -13,6 +13,8 @@ import { CopilotClient } from "../../../src"; import { CapiProxy } from "./CapiProxy"; import { retry } from "./sdkTestHelper"; +export const isCI = process.env.GITHUB_ACTIONS === "true"; + const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const SNAPSHOTS_DIR = resolve(__dirname, "../../../../test/snapshots"); @@ -42,7 +44,7 @@ export async function createSdkTestContext({ logLevel: logLevel || "error", cliPath: process.env.COPILOT_CLI_PATH, // Use fake token in CI to allow cached responses without real auth - githubToken: process.env.CI === "true" ? "fake-token-for-e2e-tests" : undefined, + githubToken: isCI ? "fake-token-for-e2e-tests" : undefined, }); const harness = { homeDir, workDir, openAiEndpoint, copilotClient, env }; diff --git a/nodejs/test/e2e/session.test.ts b/nodejs/test/e2e/session.test.ts index 9d067a8ef..7a7a6d3a0 100644 --- a/nodejs/test/e2e/session.test.ts +++ b/nodejs/test/e2e/session.test.ts @@ -2,7 +2,7 @@ import { rm } from "fs/promises"; import { describe, expect, it, onTestFinished } from "vitest"; import { ParsedHttpExchange } from "../../../test/harness/replayingCapiProxy.js"; import { CopilotClient, approveAll } from "../../src/index.js"; -import { createSdkTestContext } from "./harness/sdkTestContext.js"; +import { createSdkTestContext, isCI } from "./harness/sdkTestContext.js"; import { getFinalAssistantMessage, getNextEventOfType } from "./harness/sdkTestHelper.js"; describe("Sessions", async () => { @@ -175,6 +175,12 @@ describe("Sessions", async () => { const messages = await session2.getMessages(); const assistantMessages = messages.filter((m) => m.type === "assistant.message"); expect(assistantMessages[assistantMessages.length - 1].data.content).toContain("2"); + + // Can continue the conversation statefully + const secondAssistantMessage = await session2.sendAndWait({ + prompt: "Now if you double that, what do you get?", + }); + expect(secondAssistantMessage?.data.content).toContain("4"); }); it("should resume a session using a new client", async () => { @@ -187,7 +193,7 @@ describe("Sessions", async () => { // Resume using a new client const newClient = new CopilotClient({ env, - githubToken: process.env.CI === "true" ? "fake-token-for-e2e-tests" : undefined, + githubToken: isCI ? "fake-token-for-e2e-tests" : undefined, }); onTestFinished(() => newClient.forceStop()); @@ -202,6 +208,12 @@ describe("Sessions", async () => { const messages = await session2.getMessages(); expect(messages).toContainEqual(expect.objectContaining({ type: "user.message" })); expect(messages).toContainEqual(expect.objectContaining({ type: "session.resume" })); + + // Can continue the conversation statefully + const secondAssistantMessage = await session2.sendAndWait({ + prompt: "Now if you double that, what do you get?", + }); + expect(secondAssistantMessage?.data.content).toContain("4"); }); it("should throw error when resuming non-existent session", async () => { @@ -284,56 +296,6 @@ describe("Sessions", async () => { expect(answer?.data.content).toContain("4"); }); - it("should receive streaming delta events when streaming is enabled", async () => { - const session = await client.createSession({ - onPermissionRequest: approveAll, - streaming: true, - }); - - const deltaContents: string[] = []; - let _finalMessage: string | undefined; - - // Set up event listener before sending - const unsubscribe = session.on((event) => { - if (event.type === "assistant.message_delta") { - const delta = (event.data as { deltaContent?: string }).deltaContent; - if (delta) { - deltaContents.push(delta); - } - } else if (event.type === "assistant.message") { - _finalMessage = event.data.content; - } - }); - - const assistantMessage = await session.sendAndWait({ prompt: "What is 2+2?" }); - - unsubscribe(); - - // Should have received delta events - expect(deltaContents.length).toBeGreaterThan(0); - - // Accumulated deltas should equal the final message - const accumulated = deltaContents.join(""); - expect(accumulated).toBe(assistantMessage?.data.content); - - // Final message should contain the answer - expect(assistantMessage?.data.content).toContain("4"); - }); - - it("should pass streaming option to session creation", async () => { - // Verify that the streaming option is accepted without errors - const session = await client.createSession({ - onPermissionRequest: approveAll, - streaming: true, - }); - - expect(session.sessionId).toMatch(/^[a-f0-9-]+$/); - - // Session should still work normally - const assistantMessage = await session.sendAndWait({ prompt: "What is 1+1?" }); - expect(assistantMessage?.data.content).toContain("2"); - }); - it("should receive session events", async () => { const session = await client.createSession({ onPermissionRequest: approveAll }); const receivedEvents: Array<{ type: string }> = []; diff --git a/nodejs/test/e2e/streaming_fidelity.test.ts b/nodejs/test/e2e/streaming_fidelity.test.ts index a5a2ead26..736c9313d 100644 --- a/nodejs/test/e2e/streaming_fidelity.test.ts +++ b/nodejs/test/e2e/streaming_fidelity.test.ts @@ -2,12 +2,12 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ -import { describe, expect, it } from "vitest"; -import { SessionEvent, approveAll } from "../../src/index.js"; -import { createSdkTestContext } from "./harness/sdkTestContext"; +import { describe, expect, it, onTestFinished } from "vitest"; +import { CopilotClient, SessionEvent, approveAll } from "../../src/index.js"; +import { createSdkTestContext, isCI } from "./harness/sdkTestContext"; describe("Streaming Fidelity", async () => { - const { copilotClient: client } = await createSdkTestContext(); + const { copilotClient: client, env } = await createSdkTestContext(); it("should produce delta events when streaming is enabled", async () => { const session = await client.createSession({ @@ -71,4 +71,43 @@ describe("Streaming Fidelity", async () => { await session.destroy(); }); + + it("should produce deltas after session resume", async () => { + const session = await client.createSession({ + onPermissionRequest: approveAll, + streaming: false, + }); + await session.sendAndWait({ prompt: "What is 3 + 6?" }); + await session.destroy(); + + // Resume using a new client + const newClient = new CopilotClient({ + env, + githubToken: isCI ? "fake-token-for-e2e-tests" : undefined, + }); + onTestFinished(() => newClient.forceStop()); + const session2 = await newClient.resumeSession(session.sessionId, { + onPermissionRequest: approveAll, + streaming: true, + }); + const events: SessionEvent[] = []; + session2.on((event) => events.push(event)); + + const secondAssistantMessage = await session2.sendAndWait({ + prompt: "Now if you double that, what do you get?", + }); + expect(secondAssistantMessage?.data.content).toContain("18"); + + // Should have streaming deltas before the final message + const deltaEvents = events.filter((e) => e.type === "assistant.message_delta"); + expect(deltaEvents.length).toBeGreaterThanOrEqual(1); + + // Deltas should have content + for (const delta of deltaEvents) { + expect(delta.data.deltaContent).toBeDefined(); + expect(typeof delta.data.deltaContent).toBe("string"); + } + + await session2.destroy(); + }); }); diff --git a/python/e2e/test_session.py b/python/e2e/test_session.py index 47cb1b5ae..e6e4b303c 100644 --- a/python/e2e/test_session.py +++ b/python/e2e/test_session.py @@ -172,6 +172,13 @@ async def test_should_resume_a_session_using_the_same_client(self, ctx: E2ETestC answer2 = await get_final_assistant_message(session2) assert "2" in answer2.data.content + # Can continue the conversation statefully + answer3 = await session2.send_and_wait( + {"prompt": "Now if you double that, what do you get?"} + ) + assert answer3 is not None + assert "4" in answer3.data.content + async def test_should_resume_a_session_using_a_new_client(self, ctx: E2ETestContext): # Create initial session session1 = await ctx.client.create_session( @@ -183,7 +190,9 @@ async def test_should_resume_a_session_using_a_new_client(self, ctx: E2ETestCont assert "2" in answer.data.content # Resume using a new client - github_token = "fake-token-for-e2e-tests" if os.environ.get("CI") == "true" else None + github_token = ( + "fake-token-for-e2e-tests" if os.environ.get("GITHUB_ACTIONS") == "true" else None + ) new_client = CopilotClient( { "cli_path": ctx.cli_path, @@ -199,13 +208,17 @@ async def test_should_resume_a_session_using_a_new_client(self, ctx: E2ETestCont ) assert session2.session_id == session_id - # TODO: There's an inconsistency here. When resuming with a new client, - # we don't see the session.idle message in the history, which means we - # can't use get_final_assistant_message. messages = await session2.get_messages() message_types = [m.type.value for m in messages] assert "user.message" in message_types assert "session.resume" in message_types + + # Can continue the conversation statefully + answer2 = await session2.send_and_wait( + {"prompt": "Now if you double that, what do you get?"} + ) + assert answer2 is not None + assert "4" in answer2.data.content finally: await new_client.force_stop() @@ -417,65 +430,6 @@ async def test_should_abort_a_session(self, ctx: E2ETestContext): answer = await session.send_and_wait({"prompt": "What is 2+2?"}) assert "4" in answer.data.content - async def test_should_receive_streaming_delta_events_when_streaming_is_enabled( - self, ctx: E2ETestContext - ): - import asyncio - - session = await ctx.client.create_session( - {"streaming": True, "on_permission_request": PermissionHandler.approve_all} - ) - - delta_contents = [] - done_event = asyncio.Event() - - def on_event(event): - if event.type.value == "assistant.message_delta": - delta = getattr(event.data, "delta_content", None) - if delta: - delta_contents.append(delta) - elif event.type.value == "session.idle": - done_event.set() - - session.on(on_event) - - await session.send({"prompt": "What is 2+2?"}) - - # Wait for completion - try: - await asyncio.wait_for(done_event.wait(), timeout=60) - except TimeoutError: - pytest.fail("Timed out waiting for session.idle") - - # Should have received delta events - assert len(delta_contents) > 0, "Expected to receive delta events" - - # Get the final message to compare - assistant_message = await get_final_assistant_message(session) - - # Accumulated deltas should equal the final message - accumulated = "".join(delta_contents) - assert accumulated == assistant_message.data.content, ( - f"Accumulated deltas don't match final message.\n" - f"Accumulated: {accumulated!r}\nFinal: {assistant_message.data.content!r}" - ) - - # Final message should contain the answer - assert "4" in assistant_message.data.content - - async def test_should_pass_streaming_option_to_session_creation(self, ctx: E2ETestContext): - # Verify that the streaming option is accepted without errors - session = await ctx.client.create_session( - {"streaming": True, "on_permission_request": PermissionHandler.approve_all} - ) - - assert session.session_id - - # Session should still work normally - await session.send({"prompt": "What is 1+1?"}) - assistant_message = await get_final_assistant_message(session) - assert "2" in assistant_message.data.content - async def test_should_receive_session_events(self, ctx: E2ETestContext): import asyncio diff --git a/python/e2e/test_streaming_fidelity.py b/python/e2e/test_streaming_fidelity.py new file mode 100644 index 000000000..bca24753e --- /dev/null +++ b/python/e2e/test_streaming_fidelity.py @@ -0,0 +1,114 @@ +"""E2E Streaming Fidelity Tests""" + +import os + +import pytest + +from copilot import CopilotClient, PermissionHandler + +from .testharness import E2ETestContext + +pytestmark = pytest.mark.asyncio(loop_scope="module") + + +class TestStreamingFidelity: + async def test_should_produce_delta_events_when_streaming_is_enabled(self, ctx: E2ETestContext): + session = await ctx.client.create_session( + {"streaming": True, "on_permission_request": PermissionHandler.approve_all} + ) + + events = [] + session.on(lambda event: events.append(event)) + + await session.send_and_wait({"prompt": "Count from 1 to 5, separated by commas."}) + + types = [e.type.value for e in events] + + # Should have streaming deltas before the final message + delta_events = [e for e in events if e.type.value == "assistant.message_delta"] + assert len(delta_events) >= 1 + + # Deltas should have content + for delta in delta_events: + delta_content = getattr(delta.data, "delta_content", None) + assert delta_content is not None + assert isinstance(delta_content, str) + + # Should still have a final assistant.message + assert "assistant.message" in types + + # Deltas should come before the final message + first_delta_idx = types.index("assistant.message_delta") + last_assistant_idx = len(types) - 1 - types[::-1].index("assistant.message") + assert first_delta_idx < last_assistant_idx + + await session.destroy() + + async def test_should_not_produce_deltas_when_streaming_is_disabled(self, ctx: E2ETestContext): + session = await ctx.client.create_session( + {"streaming": False, "on_permission_request": PermissionHandler.approve_all} + ) + + events = [] + session.on(lambda event: events.append(event)) + + await session.send_and_wait({"prompt": "Say 'hello world'."}) + + delta_events = [e for e in events if e.type.value == "assistant.message_delta"] + + # No deltas when streaming is off + assert len(delta_events) == 0 + + # But should still have a final assistant.message + assistant_events = [e for e in events if e.type.value == "assistant.message"] + assert len(assistant_events) >= 1 + + await session.destroy() + + async def test_should_produce_deltas_after_session_resume(self, ctx: E2ETestContext): + session = await ctx.client.create_session( + {"streaming": False, "on_permission_request": PermissionHandler.approve_all} + ) + await session.send_and_wait({"prompt": "What is 3 + 6?"}) + await session.destroy() + + # Resume using a new client + github_token = ( + "fake-token-for-e2e-tests" if os.environ.get("GITHUB_ACTIONS") == "true" else None + ) + new_client = CopilotClient( + { + "cli_path": ctx.cli_path, + "cwd": ctx.work_dir, + "env": ctx.get_env(), + "github_token": github_token, + } + ) + + try: + session2 = await new_client.resume_session( + session.session_id, + {"streaming": True, "on_permission_request": PermissionHandler.approve_all}, + ) + events = [] + session2.on(lambda event: events.append(event)) + + answer = await session2.send_and_wait( + {"prompt": "Now if you double that, what do you get?"} + ) + assert answer is not None + assert "18" in answer.data.content + + # Should have streaming deltas before the final message + delta_events = [e for e in events if e.type.value == "assistant.message_delta"] + assert len(delta_events) >= 1 + + # Deltas should have content + for delta in delta_events: + delta_content = getattr(delta.data, "delta_content", None) + assert delta_content is not None + assert isinstance(delta_content, str) + + await session2.destroy() + finally: + await new_client.force_stop() diff --git a/python/e2e/testharness/context.py b/python/e2e/testharness/context.py index eb0c44081..c03088912 100644 --- a/python/e2e/testharness/context.py +++ b/python/e2e/testharness/context.py @@ -60,7 +60,9 @@ async def setup(self): # Create the shared client (like Node.js/Go do) # Use fake token in CI to allow cached responses without real auth - github_token = "fake-token-for-e2e-tests" if os.environ.get("CI") == "true" else None + github_token = ( + "fake-token-for-e2e-tests" if os.environ.get("GITHUB_ACTIONS") == "true" else None + ) self._client = CopilotClient( { "cli_path": self.cli_path, diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index 6864731df..1a8fbc243 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -311,7 +311,8 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { // Fallback to normal proxying if no cached response found // This implicitly captures the new exchange too - if (process.env.CI === "true") { + const isCI = process.env.GITHUB_ACTIONS === "true"; + if (isCI) { await exitWithNoMatchingRequestError( options, state.testInfo, diff --git a/test/snapshots/session/should_pass_streaming_option_to_session_creation.yaml b/test/snapshots/session/should_pass_streaming_option_to_session_creation.yaml deleted file mode 100644 index 250402101..000000000 --- a/test/snapshots/session/should_pass_streaming_option_to_session_creation.yaml +++ /dev/null @@ -1,10 +0,0 @@ -models: - - claude-sonnet-4.5 -conversations: - - messages: - - role: system - content: ${system} - - role: user - content: What is 1+1? - - role: assistant - content: 1 + 1 = 2 diff --git a/test/snapshots/session/should_receive_streaming_delta_events_when_streaming_is_enabled.yaml b/test/snapshots/session/should_receive_streaming_delta_events_when_streaming_is_enabled.yaml deleted file mode 100644 index 9fe2fcd07..000000000 --- a/test/snapshots/session/should_receive_streaming_delta_events_when_streaming_is_enabled.yaml +++ /dev/null @@ -1,10 +0,0 @@ -models: - - claude-sonnet-4.5 -conversations: - - messages: - - role: system - content: ${system} - - role: user - content: What is 2+2? - - role: assistant - content: 2 + 2 = 4 diff --git a/test/snapshots/session/should_resume_a_session_using_a_new_client.yaml b/test/snapshots/session/should_resume_a_session_using_a_new_client.yaml index 250402101..bd0285837 100644 --- a/test/snapshots/session/should_resume_a_session_using_a_new_client.yaml +++ b/test/snapshots/session/should_resume_a_session_using_a_new_client.yaml @@ -8,3 +8,7 @@ conversations: content: What is 1+1? - role: assistant content: 1 + 1 = 2 + - role: user + content: Now if you double that, what do you get? + - role: assistant + content: 2 doubled is 4. diff --git a/test/snapshots/session/should_resume_a_session_using_the_same_client.yaml b/test/snapshots/session/should_resume_a_session_using_the_same_client.yaml index 250402101..b012e26ea 100644 --- a/test/snapshots/session/should_resume_a_session_using_the_same_client.yaml +++ b/test/snapshots/session/should_resume_a_session_using_the_same_client.yaml @@ -8,3 +8,7 @@ conversations: content: What is 1+1? - role: assistant content: 1 + 1 = 2 + - role: user + content: Now if you double that, what do you get? + - role: assistant + content: If you double 2, you get 4. diff --git a/test/snapshots/streaming_fidelity/should_produce_deltas_after_session_resume.yaml b/test/snapshots/streaming_fidelity/should_produce_deltas_after_session_resume.yaml new file mode 100644 index 000000000..25e10c4b1 --- /dev/null +++ b/test/snapshots/streaming_fidelity/should_produce_deltas_after_session_resume.yaml @@ -0,0 +1,14 @@ +models: + - claude-sonnet-4.5 +conversations: + - messages: + - role: system + content: ${system} + - role: user + content: What is 3 + 6? + - role: assistant + content: 3 + 6 = 9 + - role: user + content: Now if you double that, what do you get? + - role: assistant + content: 9 × 2 = 18