From c8ac17a43bfae476f0e091efb2141a737b118c66 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 26 Sep 2025 11:04:16 +0000 Subject: [PATCH 1/3] Initial plan From 2d7969e227a4e74cef3ed9f23d9694d969835694 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 26 Sep 2025 11:13:22 +0000 Subject: [PATCH 2/3] Rename workflow steps for clarity: Print Safe Outputs and Upload Safe Outputs Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/ci-doctor.lock.yml | 4 +- .github/workflows/dev.lock.yml | 4 +- pkg/cli/workflows/test-all.lock.yml | 4 +- .../test-claude-max-patch-size.lock.yml | 4 +- .../test-claude-missing-tool.lock.yml | 4 +- .../test-claude-patch-size-exceeded.lock.yml | 4 +- .../test-codex-add-issue-comment.lock.yml | 8 ++ .../test-codex-add-issue-labels.lock.yml | 8 ++ pkg/cli/workflows/test-codex-command.lock.yml | 8 ++ .../workflows/test-codex-custom-env.lock.yml | 8 ++ .../test-copilot-add-issue-comment.lock.yml | 115 +++++++++++++++++ .../test-copilot-add-issue-labels.lock.yml | 115 +++++++++++++++++ .../test-copilot-cache-memory.lock.yml | 115 +++++++++++++++++ .../workflows/test-copilot-command.lock.yml | 115 +++++++++++++++++ .../test-copilot-create-issue.lock.yml | 115 +++++++++++++++++ ...reate-pull-request-review-comment.lock.yml | 115 +++++++++++++++++ .../test-copilot-create-pull-request.lock.yml | 115 +++++++++++++++++ ...eate-repository-security-advisory.lock.yml | 115 +++++++++++++++++ .../test-copilot-markitdown-mcp.lock.yml | 115 +++++++++++++++++ .../test-copilot-max-patch-size.lock.yml | 119 +++++++++++++++++- pkg/cli/workflows/test-copilot-mcp.lock.yml | 115 +++++++++++++++++ .../test-copilot-missing-tool.lock.yml | 119 +++++++++++++++++- .../test-copilot-patch-size-exceeded.lock.yml | 119 +++++++++++++++++- ...pilot-push-to-pull-request-branch.lock.yml | 115 +++++++++++++++++ .../test-copilot-update-issue.lock.yml | 115 +++++++++++++++++ ...playwright-accessibility-contrast.lock.yml | 4 +- .../test-playwright-screenshots.lock.yml | 4 +- pkg/cli/workflows/test-safe-jobs.lock.yml | 4 +- pkg/workflow/agentic_output_test.go | 14 +-- pkg/workflow/compiler.go | 4 +- 30 files changed, 1790 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci-doctor.lock.yml b/.github/workflows/ci-doctor.lock.yml index 6f123619677..16a82bf9ab0 100644 --- a/.github/workflows/ci-doctor.lock.yml +++ b/.github/workflows/ci-doctor.lock.yml @@ -1022,7 +1022,7 @@ jobs: # Show last few lines for debugging echo "=== Last 10 lines of Claude execution log ===" tail -10 /tmp/ci-failure-doctor.log || echo "No log content available" - - name: Print Agent output + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -1040,7 +1040,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/dev.lock.yml b/.github/workflows/dev.lock.yml index 9a4f5d7edff..9af6bcf6d80 100644 --- a/.github/workflows/dev.lock.yml +++ b/.github/workflows/dev.lock.yml @@ -980,7 +980,7 @@ jobs: # Show last few lines for debugging echo "=== Last 10 lines of Copilot execution log ===" tail -10 /tmp/dev.log || echo "No log content available" - - name: Print Agent output + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -998,7 +998,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: diff --git a/pkg/cli/workflows/test-all.lock.yml b/pkg/cli/workflows/test-all.lock.yml index 0de3aec5615..07b1a3f7ad5 100644 --- a/pkg/cli/workflows/test-all.lock.yml +++ b/pkg/cli/workflows/test-all.lock.yml @@ -1457,7 +1457,7 @@ jobs: # Show last few lines for debugging echo "=== Last 10 lines of Claude execution log ===" tail -10 /tmp/comprehensive-test-agentic-workflow-poem-bot.log || echo "No log content available" - - name: Print Agent output + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -1475,7 +1475,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: diff --git a/pkg/cli/workflows/test-claude-max-patch-size.lock.yml b/pkg/cli/workflows/test-claude-max-patch-size.lock.yml index 3935da15ac4..8bfbab65f7d 100644 --- a/pkg/cli/workflows/test-claude-max-patch-size.lock.yml +++ b/pkg/cli/workflows/test-claude-max-patch-size.lock.yml @@ -950,7 +950,7 @@ jobs: # Show last few lines for debugging echo "=== Last 10 lines of Claude execution log ===" tail -10 /tmp/test-claude-patch-size-validation.log || echo "No log content available" - - name: Print Agent output + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -968,7 +968,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: diff --git a/pkg/cli/workflows/test-claude-missing-tool.lock.yml b/pkg/cli/workflows/test-claude-missing-tool.lock.yml index 3f4dda35b53..4679df5d7ce 100644 --- a/pkg/cli/workflows/test-claude-missing-tool.lock.yml +++ b/pkg/cli/workflows/test-claude-missing-tool.lock.yml @@ -1009,7 +1009,7 @@ jobs: # Show last few lines for debugging echo "=== Last 10 lines of Claude execution log ===" tail -10 /tmp/test-claude-with-missing-tool-safe-output-and-cache-memory.log || echo "No log content available" - - name: Print Agent output + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -1027,7 +1027,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: diff --git a/pkg/cli/workflows/test-claude-patch-size-exceeded.lock.yml b/pkg/cli/workflows/test-claude-patch-size-exceeded.lock.yml index 2009b9837a5..b2a197096b9 100644 --- a/pkg/cli/workflows/test-claude-patch-size-exceeded.lock.yml +++ b/pkg/cli/workflows/test-claude-patch-size-exceeded.lock.yml @@ -952,7 +952,7 @@ jobs: # Show last few lines for debugging echo "=== Last 10 lines of Claude execution log ===" tail -10 /tmp/test-claude-patch-size-limit-exceeded.log || echo "No log content available" - - name: Print Agent output + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -970,7 +970,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: diff --git a/pkg/cli/workflows/test-codex-add-issue-comment.lock.yml b/pkg/cli/workflows/test-codex-add-issue-comment.lock.yml index ce38cfac544..07c61f1674e 100644 --- a/pkg/cli/workflows/test-codex-add-issue-comment.lock.yml +++ b/pkg/cli/workflows/test-codex-add-issue-comment.lock.yml @@ -140,6 +140,14 @@ jobs: GITHUB_AW_PROMPT: /tmp/aw-prompts/prompt.txt GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-codex-add-issue-comment.log + # Show last few lines for debugging + echo "=== Last 10 lines of Codex execution log ===" + tail -10 /tmp/test-codex-add-issue-comment.log || echo "No log content available" - name: Upload MCP logs if: always() uses: actions/upload-artifact@v4 diff --git a/pkg/cli/workflows/test-codex-add-issue-labels.lock.yml b/pkg/cli/workflows/test-codex-add-issue-labels.lock.yml index d4fcdf5ef1b..f93cda4e4c1 100644 --- a/pkg/cli/workflows/test-codex-add-issue-labels.lock.yml +++ b/pkg/cli/workflows/test-codex-add-issue-labels.lock.yml @@ -140,6 +140,14 @@ jobs: GITHUB_AW_PROMPT: /tmp/aw-prompts/prompt.txt GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-codex-add-issue-labels.log + # Show last few lines for debugging + echo "=== Last 10 lines of Codex execution log ===" + tail -10 /tmp/test-codex-add-issue-labels.log || echo "No log content available" - name: Upload MCP logs if: always() uses: actions/upload-artifact@v4 diff --git a/pkg/cli/workflows/test-codex-command.lock.yml b/pkg/cli/workflows/test-codex-command.lock.yml index e059b165e40..668e879d10c 100644 --- a/pkg/cli/workflows/test-codex-command.lock.yml +++ b/pkg/cli/workflows/test-codex-command.lock.yml @@ -142,6 +142,14 @@ jobs: GITHUB_AW_PROMPT: /tmp/aw-prompts/prompt.txt GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-codex-command.log + # Show last few lines for debugging + echo "=== Last 10 lines of Codex execution log ===" + tail -10 /tmp/test-codex-command.log || echo "No log content available" - name: Upload MCP logs if: always() uses: actions/upload-artifact@v4 diff --git a/pkg/cli/workflows/test-codex-custom-env.lock.yml b/pkg/cli/workflows/test-codex-custom-env.lock.yml index 5d4c083574c..0815a1745f3 100644 --- a/pkg/cli/workflows/test-codex-custom-env.lock.yml +++ b/pkg/cli/workflows/test-codex-custom-env.lock.yml @@ -142,6 +142,14 @@ jobs: GITHUB_AW_PROMPT: /tmp/aw-prompts/prompt.txt GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_CI }} + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-codex-custom-environment-variable.log + # Show last few lines for debugging + echo "=== Last 10 lines of Codex execution log ===" + tail -10 /tmp/test-codex-custom-environment-variable.log || echo "No log content available" - name: Upload MCP logs if: always() uses: actions/upload-artifact@v4 diff --git a/pkg/cli/workflows/test-copilot-add-issue-comment.lock.yml b/pkg/cli/workflows/test-copilot-add-issue-comment.lock.yml index 19701e233ad..2c67cb100d8 100644 --- a/pkg/cli/workflows/test-copilot-add-issue-comment.lock.yml +++ b/pkg/cli/workflows/test-copilot-add-issue-comment.lock.yml @@ -121,6 +121,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-add-issue-comment.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-add-issue-comment.log || echo "No log content available" - name: Upload engine output files uses: actions/upload-artifact@v4 with: @@ -253,4 +261,111 @@ jobs: name: test-copilot-add-issue-comment.log path: /tmp/test-copilot-add-issue-comment.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-add-issue-comment.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } diff --git a/pkg/cli/workflows/test-copilot-add-issue-labels.lock.yml b/pkg/cli/workflows/test-copilot-add-issue-labels.lock.yml index 70940890aae..c799d48cd27 100644 --- a/pkg/cli/workflows/test-copilot-add-issue-labels.lock.yml +++ b/pkg/cli/workflows/test-copilot-add-issue-labels.lock.yml @@ -121,6 +121,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-add-issue-labels.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-add-issue-labels.log || echo "No log content available" - name: Upload engine output files uses: actions/upload-artifact@v4 with: @@ -253,4 +261,111 @@ jobs: name: test-copilot-add-issue-labels.log path: /tmp/test-copilot-add-issue-labels.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-add-issue-labels.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } diff --git a/pkg/cli/workflows/test-copilot-cache-memory.lock.yml b/pkg/cli/workflows/test-copilot-cache-memory.lock.yml index 9a21509fe81..dc77992d796 100644 --- a/pkg/cli/workflows/test-copilot-cache-memory.lock.yml +++ b/pkg/cli/workflows/test-copilot-cache-memory.lock.yml @@ -206,6 +206,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-with-cache-memory-file-share.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-with-cache-memory-file-share.log || echo "No log content available" - name: Upload engine output files uses: actions/upload-artifact@v4 with: @@ -338,4 +346,111 @@ jobs: name: test-copilot-with-cache-memory-file-share.log path: /tmp/test-copilot-with-cache-memory-file-share.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-with-cache-memory-file-share.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } diff --git a/pkg/cli/workflows/test-copilot-command.lock.yml b/pkg/cli/workflows/test-copilot-command.lock.yml index ed2eb2ea24b..e154452f8ca 100644 --- a/pkg/cli/workflows/test-copilot-command.lock.yml +++ b/pkg/cli/workflows/test-copilot-command.lock.yml @@ -123,6 +123,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-command.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-command.log || echo "No log content available" - name: Upload engine output files uses: actions/upload-artifact@v4 with: @@ -255,4 +263,111 @@ jobs: name: test-copilot-command.log path: /tmp/test-copilot-command.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-command.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } diff --git a/pkg/cli/workflows/test-copilot-create-issue.lock.yml b/pkg/cli/workflows/test-copilot-create-issue.lock.yml index d37f6774536..5c2c9d4018f 100644 --- a/pkg/cli/workflows/test-copilot-create-issue.lock.yml +++ b/pkg/cli/workflows/test-copilot-create-issue.lock.yml @@ -121,6 +121,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-create-issue.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-create-issue.log || echo "No log content available" - name: Upload engine output files uses: actions/upload-artifact@v4 with: @@ -253,4 +261,111 @@ jobs: name: test-copilot-create-issue.log path: /tmp/test-copilot-create-issue.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-create-issue.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } diff --git a/pkg/cli/workflows/test-copilot-create-pull-request-review-comment.lock.yml b/pkg/cli/workflows/test-copilot-create-pull-request-review-comment.lock.yml index 237df5d3dac..c8b96905300 100644 --- a/pkg/cli/workflows/test-copilot-create-pull-request-review-comment.lock.yml +++ b/pkg/cli/workflows/test-copilot-create-pull-request-review-comment.lock.yml @@ -121,6 +121,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-create-pull-request-review-comment.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-create-pull-request-review-comment.log || echo "No log content available" - name: Upload engine output files uses: actions/upload-artifact@v4 with: @@ -253,4 +261,111 @@ jobs: name: test-copilot-create-pull-request-review-comment.log path: /tmp/test-copilot-create-pull-request-review-comment.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-create-pull-request-review-comment.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } diff --git a/pkg/cli/workflows/test-copilot-create-pull-request.lock.yml b/pkg/cli/workflows/test-copilot-create-pull-request.lock.yml index 5334665233a..c00859d980b 100644 --- a/pkg/cli/workflows/test-copilot-create-pull-request.lock.yml +++ b/pkg/cli/workflows/test-copilot-create-pull-request.lock.yml @@ -128,6 +128,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-create-pull-request.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-create-pull-request.log || echo "No log content available" - name: Upload engine output files uses: actions/upload-artifact@v4 with: @@ -260,4 +268,111 @@ jobs: name: test-copilot-create-pull-request.log path: /tmp/test-copilot-create-pull-request.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-create-pull-request.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } diff --git a/pkg/cli/workflows/test-copilot-create-repository-security-advisory.lock.yml b/pkg/cli/workflows/test-copilot-create-repository-security-advisory.lock.yml index 3e8a0ec4aad..7b7f9f28570 100644 --- a/pkg/cli/workflows/test-copilot-create-repository-security-advisory.lock.yml +++ b/pkg/cli/workflows/test-copilot-create-repository-security-advisory.lock.yml @@ -124,6 +124,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-create-repository-security-advisory.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-create-repository-security-advisory.log || echo "No log content available" - name: Upload engine output files uses: actions/upload-artifact@v4 with: @@ -256,4 +264,111 @@ jobs: name: test-copilot-create-repository-security-advisory.log path: /tmp/test-copilot-create-repository-security-advisory.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-create-repository-security-advisory.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } diff --git a/pkg/cli/workflows/test-copilot-markitdown-mcp.lock.yml b/pkg/cli/workflows/test-copilot-markitdown-mcp.lock.yml index 2a722b41cec..9e745a010a0 100644 --- a/pkg/cli/workflows/test-copilot-markitdown-mcp.lock.yml +++ b/pkg/cli/workflows/test-copilot-markitdown-mcp.lock.yml @@ -137,6 +137,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-markitdown-mcp.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-markitdown-mcp.log || echo "No log content available" - name: Upload engine output files uses: actions/upload-artifact@v4 with: @@ -269,4 +277,111 @@ jobs: name: test-copilot-markitdown-mcp.log path: /tmp/test-copilot-markitdown-mcp.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-markitdown-mcp.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } diff --git a/pkg/cli/workflows/test-copilot-max-patch-size.lock.yml b/pkg/cli/workflows/test-copilot-max-patch-size.lock.yml index 2c502e6f6e6..f8f1cfc652b 100644 --- a/pkg/cli/workflows/test-copilot-max-patch-size.lock.yml +++ b/pkg/cli/workflows/test-copilot-max-patch-size.lock.yml @@ -766,7 +766,15 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ - - name: Print Agent output + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-patch-size-validation.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-patch-size-validation.log || echo "No log content available" + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -784,7 +792,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: @@ -1660,6 +1668,113 @@ jobs: name: test-copilot-patch-size-validation.log path: /tmp/test-copilot-patch-size-validation.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-patch-size-validation.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } - name: Generate git patch if: always() env: diff --git a/pkg/cli/workflows/test-copilot-mcp.lock.yml b/pkg/cli/workflows/test-copilot-mcp.lock.yml index e5c1d940043..e58da88aa6c 100644 --- a/pkg/cli/workflows/test-copilot-mcp.lock.yml +++ b/pkg/cli/workflows/test-copilot-mcp.lock.yml @@ -126,6 +126,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-mcp.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-mcp.log || echo "No log content available" - name: Upload engine output files uses: actions/upload-artifact@v4 with: @@ -258,4 +266,111 @@ jobs: name: test-copilot-mcp.log path: /tmp/test-copilot-mcp.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-mcp.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } diff --git a/pkg/cli/workflows/test-copilot-missing-tool.lock.yml b/pkg/cli/workflows/test-copilot-missing-tool.lock.yml index c0e7cd47046..9aee604b8a1 100644 --- a/pkg/cli/workflows/test-copilot-missing-tool.lock.yml +++ b/pkg/cli/workflows/test-copilot-missing-tool.lock.yml @@ -823,7 +823,15 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ - - name: Print Agent output + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-with-missing-tool-safe-output-and-cache-memory.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-with-missing-tool-safe-output-and-cache-memory.log || echo "No log content available" + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -841,7 +849,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: @@ -1717,6 +1725,113 @@ jobs: name: test-copilot-with-missing-tool-safe-output-and-cache-memory.log path: /tmp/test-copilot-with-missing-tool-safe-output-and-cache-memory.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-with-missing-tool-safe-output-and-cache-memory.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } missing_tool: needs: agent diff --git a/pkg/cli/workflows/test-copilot-patch-size-exceeded.lock.yml b/pkg/cli/workflows/test-copilot-patch-size-exceeded.lock.yml index 1f66552f122..85ee80e4e0e 100644 --- a/pkg/cli/workflows/test-copilot-patch-size-exceeded.lock.yml +++ b/pkg/cli/workflows/test-copilot-patch-size-exceeded.lock.yml @@ -768,7 +768,15 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ - - name: Print Agent output + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-patch-size-limit-exceeded.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-patch-size-limit-exceeded.log || echo "No log content available" + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -786,7 +794,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: @@ -1662,6 +1670,113 @@ jobs: name: test-copilot-patch-size-limit-exceeded.log path: /tmp/test-copilot-patch-size-limit-exceeded.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-patch-size-limit-exceeded.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } - name: Generate git patch if: always() env: diff --git a/pkg/cli/workflows/test-copilot-push-to-pull-request-branch.lock.yml b/pkg/cli/workflows/test-copilot-push-to-pull-request-branch.lock.yml index c495f6b8aa5..1f23e275d77 100644 --- a/pkg/cli/workflows/test-copilot-push-to-pull-request-branch.lock.yml +++ b/pkg/cli/workflows/test-copilot-push-to-pull-request-branch.lock.yml @@ -128,6 +128,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-push-to-pr-branch.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-push-to-pr-branch.log || echo "No log content available" - name: Upload engine output files uses: actions/upload-artifact@v4 with: @@ -260,4 +268,111 @@ jobs: name: test-copilot-push-to-pr-branch.log path: /tmp/test-copilot-push-to-pr-branch.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-push-to-pr-branch.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } diff --git a/pkg/cli/workflows/test-copilot-update-issue.lock.yml b/pkg/cli/workflows/test-copilot-update-issue.lock.yml index 576bb05fac2..36e2aaa9677 100644 --- a/pkg/cli/workflows/test-copilot-update-issue.lock.yml +++ b/pkg/cli/workflows/test-copilot-update-issue.lock.yml @@ -124,6 +124,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} XDG_CONFIG_HOME: /tmp/.copilot/ XDG_STATE_HOME: /tmp/.copilot/ + - name: Ensure log file exists + if: always() + run: | + # Ensure log file exists + touch /tmp/test-copilot-update-issue.log + # Show last few lines for debugging + echo "=== Last 10 lines of Copilot execution log ===" + tail -10 /tmp/test-copilot-update-issue.log || echo "No log content available" - name: Upload engine output files uses: actions/upload-artifact@v4 with: @@ -256,4 +264,111 @@ jobs: name: test-copilot-update-issue.log path: /tmp/test-copilot-update-issue.log if-no-files-found: warn + - name: Validate agent logs for errors + if: always() + uses: actions/github-script@v8 + env: + GITHUB_AW_AGENT_OUTPUT: /tmp/test-copilot-update-issue.log + GITHUB_AW_ERROR_PATTERNS: "[{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(ERROR)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped ERROR messages\"},{\"pattern\":\"(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\s+\\\\[(WARN|WARNING)\\\\]\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI timestamped WARNING messages\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(CRITICAL|ERROR):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed critical/error messages with timestamp\"},{\"pattern\":\"\\\\[(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}Z)\\\\]\\\\s+(WARNING):\\\\s+(.+)\",\"level_group\":2,\"message_group\":3,\"description\":\"Copilot CLI bracketed warning messages with timestamp\"},{\"pattern\":\"(Error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic error messages from Copilot CLI or Node.js\"},{\"pattern\":\"npm ERR!\\\\s+(.+)\",\"level_group\":0,\"message_group\":1,\"description\":\"NPM error messages during Copilot CLI installation or execution\"},{\"pattern\":\"(Warning):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Generic warning messages from Copilot CLI\"},{\"pattern\":\"(Fatal error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Fatal error messages from Copilot CLI\"},{\"pattern\":\"copilot:\\\\s+(error):\\\\s+(.+)\",\"level_group\":1,\"message_group\":2,\"description\":\"Copilot CLI command-level error messages\"}]" + with: + script: | + function main() { + const fs = require("fs"); + try { + const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; + if (!logFile) { + throw new Error("GITHUB_AW_AGENT_OUTPUT environment variable is required"); + } + if (!fs.existsSync(logFile)) { + throw new Error(`Log file not found: ${logFile}`); + } + const patterns = getErrorPatternsFromEnv(); + if (patterns.length === 0) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required and must contain at least one pattern"); + } + const content = fs.readFileSync(logFile, "utf8"); + const hasErrors = validateErrors(content, patterns); + if (hasErrors) { + core.setFailed("Errors detected in agent logs - failing workflow step"); + } else { + core.info("Error validation completed successfully"); + } + } catch (error) { + console.debug(error); + core.setFailed(`Error validating log: ${error instanceof Error ? error.message : String(error)}`); + } + } + function getErrorPatternsFromEnv() { + const patternsEnv = process.env.GITHUB_AW_ERROR_PATTERNS; + if (!patternsEnv) { + throw new Error("GITHUB_AW_ERROR_PATTERNS environment variable is required"); + } + try { + const patterns = JSON.parse(patternsEnv); + if (!Array.isArray(patterns)) { + throw new Error("GITHUB_AW_ERROR_PATTERNS must be a JSON array"); + } + return patterns; + } catch (e) { + throw new Error(`Failed to parse GITHUB_AW_ERROR_PATTERNS as JSON: ${e instanceof Error ? e.message : String(e)}`); + } + } + function validateErrors(logContent, patterns) { + const lines = logContent.split("\n"); + let hasErrors = false; + for (const pattern of patterns) { + const regex = new RegExp(pattern.pattern, "g"); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + let match; + while ((match = regex.exec(line)) !== null) { + const level = extractLevel(match, pattern); + const message = extractMessage(match, pattern, line); + const errorMessage = `Line ${lineIndex + 1}: ${message} (Pattern: ${pattern.description || "Unknown pattern"}, Raw log: ${truncateString(line.trim(), 120)})`; + if (level.toLowerCase() === "error") { + core.error(errorMessage); + hasErrors = true; + } else { + core.warning(errorMessage); + } + } + } + } + return hasErrors; + } + function extractLevel(match, pattern) { + if (pattern.level_group && pattern.level_group > 0 && match[pattern.level_group]) { + return match[pattern.level_group]; + } + const fullMatch = match[0]; + if (fullMatch.toLowerCase().includes("error")) { + return "error"; + } else if (fullMatch.toLowerCase().includes("warn")) { + return "warning"; + } + return "unknown"; + } + function extractMessage(match, pattern, fullLine) { + if (pattern.message_group && pattern.message_group > 0 && match[pattern.message_group]) { + return match[pattern.message_group].trim(); + } + return match[0] || fullLine.trim(); + } + function truncateString(str, maxLength) { + if (!str) return ""; + if (str.length <= maxLength) return str; + return str.substring(0, maxLength) + "..."; + } + if (typeof module !== "undefined" && module.exports) { + module.exports = { + validateErrors, + extractLevel, + extractMessage, + getErrorPatternsFromEnv, + truncateString, + }; + } + if (typeof module === "undefined" || require.main === module) { + main(); + } diff --git a/pkg/cli/workflows/test-playwright-accessibility-contrast.lock.yml b/pkg/cli/workflows/test-playwright-accessibility-contrast.lock.yml index b915c7d1fe3..4dfc636e54f 100644 --- a/pkg/cli/workflows/test-playwright-accessibility-contrast.lock.yml +++ b/pkg/cli/workflows/test-playwright-accessibility-contrast.lock.yml @@ -967,7 +967,7 @@ jobs: # Show last few lines for debugging echo "=== Last 10 lines of Claude execution log ===" tail -10 /tmp/test-playwright-accessibility-contrast.log || echo "No log content available" - - name: Print Agent output + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -985,7 +985,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: diff --git a/pkg/cli/workflows/test-playwright-screenshots.lock.yml b/pkg/cli/workflows/test-playwright-screenshots.lock.yml index 3df33fdfe24..c04b4d8f9a1 100644 --- a/pkg/cli/workflows/test-playwright-screenshots.lock.yml +++ b/pkg/cli/workflows/test-playwright-screenshots.lock.yml @@ -1115,7 +1115,7 @@ jobs: # Show last few lines for debugging echo "=== Last 10 lines of Claude execution log ===" tail -10 /tmp/dev.log || echo "No log content available" - - name: Print Agent output + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -1133,7 +1133,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: diff --git a/pkg/cli/workflows/test-safe-jobs.lock.yml b/pkg/cli/workflows/test-safe-jobs.lock.yml index 30c7b6bb4ed..a317c032115 100644 --- a/pkg/cli/workflows/test-safe-jobs.lock.yml +++ b/pkg/cli/workflows/test-safe-jobs.lock.yml @@ -1153,7 +1153,7 @@ jobs: # Show last few lines for debugging echo "=== Last 10 lines of Claude execution log ===" tail -10 /tmp/test-safe-jobs.log || echo "No log content available" - - name: Print Agent output + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -1171,7 +1171,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: diff --git a/pkg/workflow/agentic_output_test.go b/pkg/workflow/agentic_output_test.go index 31f4af80f32..7b20525635e 100644 --- a/pkg/workflow/agentic_output_test.go +++ b/pkg/workflow/agentic_output_test.go @@ -69,8 +69,8 @@ This workflow tests the agentic output collection functionality. t.Error("Expected 'Ingest agent output' step to be in generated workflow") } - if !strings.Contains(lockContent, "- name: Upload agentic output file") { - t.Error("Expected 'Upload agentic output file' step to be in generated workflow") + if !strings.Contains(lockContent, "- name: Upload Safe Outputs") { + t.Error("Expected 'Upload Safe Outputs' step to be in generated workflow") } if !strings.Contains(lockContent, "- name: Upload sanitized agent output") { @@ -101,13 +101,13 @@ This workflow tests the agentic output collection functionality. t.Error("Claude workflow should NOT reference 'agent_outputs' artifact (Claude CLI no longer produces output.txt)") } - // Verify Print Agent output step has file existence check + // Verify Print Safe Outputs step has file existence check if !strings.Contains(lockContent, "if [ -f ${{ env.GITHUB_AW_SAFE_OUTPUTS }} ]; then") { - t.Error("Expected Print Agent output step to check if output file exists before reading it") + t.Error("Expected Print Safe Outputs step to check if output file exists before reading it") } if !strings.Contains(lockContent, "No agent output file found") { - t.Error("Expected Print Agent output step to provide message when no output file found") + t.Error("Expected Print Safe Outputs step to provide message when no output file found") } // Verify that both artifacts are uploaded @@ -177,8 +177,8 @@ This workflow tests that Codex engine gets GITHUB_AW_SAFE_OUTPUTS but not engine t.Error("Codex workflow should have 'Ingest agent output' step (GITHUB_AW_SAFE_OUTPUTS functionality)") } - if !strings.Contains(lockContent, "- name: Upload agentic output file") { - t.Error("Codex workflow should have 'Upload agentic output file' step (GITHUB_AW_SAFE_OUTPUTS functionality)") + if !strings.Contains(lockContent, "- name: Upload Safe Outputs") { + t.Error("Codex workflow should have 'Upload Safe Outputs' step (GITHUB_AW_SAFE_OUTPUTS functionality)") } if !strings.Contains(lockContent, "- name: Upload sanitized agent output") { diff --git a/pkg/workflow/compiler.go b/pkg/workflow/compiler.go index eae74f2294e..705c9c96bd7 100644 --- a/pkg/workflow/compiler.go +++ b/pkg/workflow/compiler.go @@ -2273,7 +2273,7 @@ func (c *Compiler) generateSafeOutputsConfig(data *WorkflowData) string { // generateOutputCollectionStep generates a step that reads the output file and sets it as a GitHub Actions output func (c *Compiler) generateOutputCollectionStep(yaml *strings.Builder, data *WorkflowData) { - yaml.WriteString(" - name: Print Agent output\n") + yaml.WriteString(" - name: Print Safe Outputs\n") yaml.WriteString(" env:\n") yaml.WriteString(" GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }}\n") yaml.WriteString(" run: |\n") @@ -2292,7 +2292,7 @@ func (c *Compiler) generateOutputCollectionStep(yaml *strings.Builder, data *Wor yaml.WriteString(" echo '``````' >> $GITHUB_STEP_SUMMARY\n") yaml.WriteString(" echo \"\" >> $GITHUB_STEP_SUMMARY\n") - yaml.WriteString(" - name: Upload agentic output file\n") + yaml.WriteString(" - name: Upload Safe Outputs\n") yaml.WriteString(" if: always()\n") yaml.WriteString(" uses: actions/upload-artifact@v4\n") yaml.WriteString(" with:\n") From 7db049936bd39343052a606245da3dfbe82294e8 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Fri, 26 Sep 2025 11:16:07 +0000 Subject: [PATCH 3/3] rebuild --- .github/workflows/tidy.lock.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tidy.lock.yml b/.github/workflows/tidy.lock.yml index 694d40671e1..7e34b6fb010 100644 --- a/.github/workflows/tidy.lock.yml +++ b/.github/workflows/tidy.lock.yml @@ -914,7 +914,7 @@ jobs: # Show last few lines for debugging echo "=== Last 10 lines of Copilot execution log ===" tail -10 /tmp/code-tidying-agent.log || echo "No log content available" - - name: Print Agent output + - name: Print Safe Outputs env: GITHUB_AW_SAFE_OUTPUTS: ${{ env.GITHUB_AW_SAFE_OUTPUTS }} run: | @@ -932,7 +932,7 @@ jobs: fi echo '``````' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - name: Upload agentic output file + - name: Upload Safe Outputs if: always() uses: actions/upload-artifact@v4 with: