diff --git a/.github/workflows/daily-fact.lock.yml b/.github/workflows/daily-fact.lock.yml index eaa4424d84d..d2918239aaf 100644 --- a/.github/workflows/daily-fact.lock.yml +++ b/.github/workflows/daily-fact.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"af7c61e76abdaae9d6d291e56f257ddcba114e4dc0595519936f5beadafcffec","strict":true,"agent_id":"codex","agent_model":"gpt-5.4-mini"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"36d37a405b99aa890e631983125c296a231d492607435c20e8632d45621f8ecd","strict":true,"agent_id":"codex","agent_model":"gpt-5.4-mini"} # gh-aw-manifest: {"version":1,"secrets":["CODEX_API_KEY","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_ENDPOINT","GH_AW_OTEL_HEADERS","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/setup-python","sha":"a309ff8b426b58ec0e2a45f0f869d46889d02405","version":"v6.2.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.42"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.42"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.25.42"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.42"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.6","digest":"sha256:2bb8eef86006a4c5963c55616a9c51c32f27bfdecb023b8aa6f91f6718d9171c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.6@sha256:2bb8eef86006a4c5963c55616a9c51c32f27bfdecb023b8aa6f91f6718d9171c"},{"image":"ghcr.io/github/github-mcp-server:v1.0.3","digest":"sha256:2ac27ef03461ef2b877031b838a7d1fd7f12b12d4ace7796d8cad91446d55959","pinned_image":"ghcr.io/github/github-mcp-server:v1.0.3@sha256:2ac27ef03461ef2b877031b838a7d1fd7f12b12d4ace7796d8cad91446d55959"},{"image":"node:lts-alpine","digest":"sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f","pinned_image":"node:lts-alpine@sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f"}]} # ___ _ _ # / _ \ | | (_) @@ -97,6 +97,7 @@ jobs: comment_id: "" comment_repo: "" engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} + experiments: ${{ steps.pick-experiment.outputs.experiments }} lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} model: ${{ steps.generate_aw_info.outputs.model }} secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} @@ -189,10 +190,46 @@ jobs: setupGlobals(core, github, context, exec, io, getOctokit); const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); await main(); + - name: Restore experiment state from git + id: restore-experiment-state + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.json + GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments + GH_AW_EXPERIMENT_BRANCH: experiments/dailyfact + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/load_experiment_state_from_repo.cjs'); + await main(); + - name: Pick experiment variants + id: pick-experiment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EXPERIMENT_SPEC: '{"reasoning_depth":{"variants":["single_pass","multi_candidate"],"description":"Tests whether deliberating over multiple candidate facts before writing improves verse novelty and engagement.","hypothesis":"H0: no change in discussion engagement rate. H1: multi_candidate produces more novel verses with higher reaction counts (expected +20% reactions).","metric":"discussion_reaction_count","secondary_metrics":["output_length_chars","run_duration_ms"],"guardrail_metrics":[{"name":"empty_output_rate","threshold":"\u003c0.05"},{"name":"run_success_rate","threshold":"\u003e=0.95"}],"min_samples":30,"weight":[50,50],"issue":31324,"start_date":"2026-05-11"}}' + GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.json + GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/pick_experiment.cjs'); + await main(); + - name: Upload experiment artifact + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dailyfact-experiment + path: /tmp/gh-aw/experiments + if-no-files-found: ignore + retention-days: 30 - name: Create prompt with built-in context env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_EXPERIMENTS_REASONING_DEPTH: ${{ steps.pick-experiment.outputs.reasoning_depth }} + GH_AW_EXPR_DD64DF46: ${{ steps.pick-experiment.outputs.reasoning_depth == "multi_candidate" }} GH_AW_GITHUB_ACTOR: ${{ github.actor }} GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} @@ -205,21 +242,21 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_291a7ecc0036fade_EOF' + cat << 'GH_AW_PROMPT_6f6003e03d2c7d0e_EOF' - GH_AW_PROMPT_291a7ecc0036fade_EOF + GH_AW_PROMPT_6f6003e03d2c7d0e_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/cache_memory_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_291a7ecc0036fade_EOF' + cat << 'GH_AW_PROMPT_6f6003e03d2c7d0e_EOF' Tools: add_comment, missing_tool, missing_data, noop - GH_AW_PROMPT_291a7ecc0036fade_EOF + GH_AW_PROMPT_6f6003e03d2c7d0e_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_291a7ecc0036fade_EOF' + cat << 'GH_AW_PROMPT_6f6003e03d2c7d0e_EOF' The following GitHub context information is available for this workflow: {{#if __GH_AW_GITHUB_ACTOR__ }} @@ -248,9 +285,9 @@ jobs: {{/if}} - GH_AW_PROMPT_291a7ecc0036fade_EOF + GH_AW_PROMPT_6f6003e03d2c7d0e_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/cli_proxy_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_291a7ecc0036fade_EOF' + cat << 'GH_AW_PROMPT_6f6003e03d2c7d0e_EOF' @@ -297,7 +334,11 @@ jobs: ## Guidelines - **Check memory first**: Skip any PR, issue, or release that already appears in the palace results from Step 0 + {{#if __GH_AW_EXPR_DD64DF46__ }} + - **Multi-candidate deliberation**: Before writing, identify exactly **3 distinct candidate facts** (one PR, one issue or release, one contributor or pattern). For each candidate write one sentence on why it is novel today. Then score each candidate 1–5 on: (a) novelty vs palace memory, (b) intrinsic poetic potential. Select the highest-scoring candidate and write the verse for that one only. + {{else}} - **Favor recent updates** but include variety - pick something interesting, not just the most recent + {{/if}} - **Be specific**: Include PR numbers, issue references, or release tags when relevant - **Keep it short**: One or two poetic sentences for the main fact, optionally with a brief context - **Be poetic**: Use lyrical, whimsical language that celebrates the beauty of code and collaboration @@ -349,13 +390,15 @@ jobs: {{#runtime-import shared/noop-reminder.md}} - GH_AW_PROMPT_291a7ecc0036fade_EOF + GH_AW_PROMPT_6f6003e03d2c7d0e_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_ENGINE_ID: "codex" + GH_AW_EXPERIMENTS_REASONING_DEPTH: ${{ steps.pick-experiment.outputs.reasoning_depth }} + GH_AW_EXPR_DD64DF46: ${{ steps.pick-experiment.outputs.reasoning_depth == "multi_candidate" }} GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} with: script: | @@ -370,6 +413,8 @@ jobs: GH_AW_ALLOWED_EXTENSIONS: '' GH_AW_CACHE_DESCRIPTION: '' GH_AW_CACHE_DIR: '/tmp/gh-aw/cache-memory/' + GH_AW_EXPERIMENTS_REASONING_DEPTH: ${{ steps.pick-experiment.outputs.reasoning_depth }} + GH_AW_EXPR_DD64DF46: ${{ steps.pick-experiment.outputs.reasoning_depth == "multi_candidate" }} GH_AW_GITHUB_ACTOR: ${{ github.actor }} GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} @@ -393,6 +438,8 @@ jobs: GH_AW_ALLOWED_EXTENSIONS: process.env.GH_AW_ALLOWED_EXTENSIONS, GH_AW_CACHE_DESCRIPTION: process.env.GH_AW_CACHE_DESCRIPTION, GH_AW_CACHE_DIR: process.env.GH_AW_CACHE_DIR, + GH_AW_EXPERIMENTS_REASONING_DEPTH: process.env.GH_AW_EXPERIMENTS_REASONING_DEPTH, + GH_AW_EXPR_DD64DF46: process.env.GH_AW_EXPR_DD64DF46, GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, @@ -598,9 +645,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_624158a7bed00a1d_EOF' + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_d1209a06df4079df_EOF' {"add_comment":{"max":1,"target":"4750"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_624158a7bed00a1d_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_d1209a06df4079df_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -787,7 +834,7 @@ jobs: DOCKER_SOCK_GID=$(stat -c '%g' /var/run/docker.sock 2>/dev/null || echo '0') export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -e GITHUB_AW_OTEL_TRACE_ID -e GITHUB_AW_OTEL_PARENT_SPAN_ID -e CODEX_HOME -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.3.6' - cat > "${RUNNER_TEMP}/gh-aw/mcp-config/config.toml" << GH_AW_MCP_CONFIG_df65c8ac8eef6bf5_EOF + cat > "${RUNNER_TEMP}/gh-aw/mcp-config/config.toml" << GH_AW_MCP_CONFIG_ece71de4bad38321_EOF [history] persistence = "none" @@ -814,11 +861,11 @@ jobs: [mcp_servers.safeoutputs."guard-policies".write-sink] accept = ["*"] - GH_AW_MCP_CONFIG_df65c8ac8eef6bf5_EOF + GH_AW_MCP_CONFIG_ece71de4bad38321_EOF # Generate JSON config for MCP gateway GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_df65c8ac8eef6bf5_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_ece71de4bad38321_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "mempalace": { @@ -881,11 +928,11 @@ jobs: } } } - GH_AW_MCP_CONFIG_df65c8ac8eef6bf5_EOF + GH_AW_MCP_CONFIG_ece71de4bad38321_EOF # Sync converter output to writable CODEX_HOME for Codex mkdir -p /tmp/gh-aw/mcp-config - cat > "/tmp/gh-aw/mcp-config/config.toml" << GH_AW_CODEX_SHELL_POLICY_01ccd543b3815814_EOF + cat > "/tmp/gh-aw/mcp-config/config.toml" << GH_AW_CODEX_SHELL_POLICY_50e3008d43a21862_EOF model_provider = "openai-proxy" @@ -897,7 +944,7 @@ jobs: [shell_environment_policy] inherit = "core" include_only = ["CODEX_API_KEY", "GH_AW_ASSETS_ALLOWED_EXTS", "GH_AW_ASSETS_BRANCH", "GH_AW_ASSETS_MAX_SIZE_KB", "GH_AW_SAFE_OUTPUTS", "GITHUB_REPOSITORY", "GITHUB_SERVER_URL", "HOME", "OPENAI_API_KEY", "PATH"] - GH_AW_CODEX_SHELL_POLICY_01ccd543b3815814_EOF + GH_AW_CODEX_SHELL_POLICY_50e3008d43a21862_EOF awk ' BEGIN { skip_openai_proxy = 0 } /^[[:space:]]*model_provider[[:space:]]*=/ { next } @@ -1148,6 +1195,7 @@ jobs: - activation - agent - detection + - push_experiments_state - safe_outputs - update_cache_memory if: > @@ -1349,6 +1397,12 @@ jobs: mkdir -p /tmp/gh-aw/ find "/tmp/gh-aw/" -type f -print echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dailyfact-experiment + path: /tmp/gh-aw/experiments/ - name: Checkout repository for patch context if: needs.agent.outputs.has_patch == 'true' uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -1448,18 +1502,18 @@ jobs: DOCKER_SOCK_GID=$(stat -c '%g' /var/run/docker.sock 2>/dev/null || echo '0') export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e CODEX_HOME -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.3.6' - cat > "${RUNNER_TEMP}/gh-aw/mcp-config/config.toml" << GH_AW_MCP_CONFIG_0a43f5caf239327a_EOF + cat > "${RUNNER_TEMP}/gh-aw/mcp-config/config.toml" << GH_AW_MCP_CONFIG_d3684e3b46c26350_EOF [history] persistence = "none" [shell_environment_policy] inherit = "core" include_only = ["CODEX_API_KEY", "HOME", "OPENAI_API_KEY", "PATH"] - GH_AW_MCP_CONFIG_0a43f5caf239327a_EOF + GH_AW_MCP_CONFIG_d3684e3b46c26350_EOF # Generate JSON config for MCP gateway GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_2074415d4870b663_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_6e95c2a80ec5cd9d_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { }, @@ -1470,11 +1524,11 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_2074415d4870b663_EOF + GH_AW_MCP_CONFIG_6e95c2a80ec5cd9d_EOF # Sync converter output to writable CODEX_HOME for Codex mkdir -p /tmp/gh-aw/mcp-config - cat > "/tmp/gh-aw/mcp-config/config.toml" << GH_AW_CODEX_SHELL_POLICY_3bc3ee739bd056b7_EOF + cat > "/tmp/gh-aw/mcp-config/config.toml" << GH_AW_CODEX_SHELL_POLICY_71d735b59c609993_EOF model_provider = "openai-proxy" [model_providers.openai-proxy] name = "OpenAI AWF proxy" @@ -1484,7 +1538,7 @@ jobs: [shell_environment_policy] inherit = "core" include_only = ["CODEX_API_KEY", "HOME", "OPENAI_API_KEY", "PATH"] - GH_AW_CODEX_SHELL_POLICY_3bc3ee739bd056b7_EOF + GH_AW_CODEX_SHELL_POLICY_71d735b59c609993_EOF awk ' BEGIN { skip_openai_proxy = 0 } /^[[:space:]]*model_provider[[:space:]]*=/ { next } @@ -1564,6 +1618,82 @@ jobs: } } + push_experiments_state: + needs: activation + if: always() && (!cancelled()) && needs.activation.result == 'success' + runs-on: ubuntu-slim + permissions: + contents: write + steps: + - name: Checkout actions folder + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: github/gh-aw + sparse-checkout: | + actions + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Daily Fact About gh-aw" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-fact.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "0.129.0" + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Download experiment artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: dailyfact-experiment + path: /tmp/gh-aw/experiments + - name: Push experiment state to git + id: push_experiments_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments + GH_AW_EXPERIMENT_BRANCH: experiments/dailyfact + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + persist-credentials: false + safe_outputs: needs: - activation diff --git a/.github/workflows/daily-fact.md b/.github/workflows/daily-fact.md index 434d3e57b89..7574d87991c 100644 --- a/.github/workflows/daily-fact.md +++ b/.github/workflows/daily-fact.md @@ -16,6 +16,22 @@ engine: model: gpt-5.4-mini bare: true strict: true +experiments: + reasoning_depth: + variants: [single_pass, multi_candidate] + description: "Tests whether deliberating over multiple candidate facts before writing improves verse novelty and engagement." + hypothesis: "H0: no change in discussion engagement rate. H1: multi_candidate produces more novel verses with higher reaction counts (expected +20% reactions)." + metric: discussion_reaction_count + secondary_metrics: [output_length_chars, run_duration_ms] + guardrail_metrics: + - name: empty_output_rate + threshold: "<0.05" + - name: run_success_rate + threshold: ">=0.95" + min_samples: 30 + weight: [50, 50] + start_date: "2026-05-11" + issue: 31324 timeout-minutes: 15 runs-on: aw-gpu-runner-T4 runtimes: @@ -83,7 +99,11 @@ Mine recent activity from the repository to find interesting facts. Focus on: ## Guidelines - **Check memory first**: Skip any PR, issue, or release that already appears in the palace results from Step 0 +{{#if experiments.reasoning_depth == "multi_candidate"}} +- **Multi-candidate deliberation**: Before writing, identify exactly **3 distinct candidate facts** (one PR, one issue or release, one contributor or pattern). For each candidate write one sentence on why it is novel today. Then score each candidate 1–5 on: (a) novelty vs palace memory, (b) intrinsic poetic potential. Select the highest-scoring candidate and write the verse for that one only. +{{else}} - **Favor recent updates** but include variety - pick something interesting, not just the most recent +{{/if}} - **Be specific**: Include PR numbers, issue references, or release tags when relevant - **Keep it short**: One or two poetic sentences for the main fact, optionally with a brief context - **Be poetic**: Use lyrical, whimsical language that celebrates the beauty of code and collaboration diff --git a/pkg/workflow/expression_extraction.go b/pkg/workflow/expression_extraction.go index ce3156c3a9d..00b92265f1c 100644 --- a/pkg/workflow/expression_extraction.go +++ b/pkg/workflow/expression_extraction.go @@ -199,6 +199,14 @@ func transformActivationOutputs(expr string) string { // experimentNameRegex matches experiments. expressions where name is a simple identifier. var experimentNameRegex = regexp.MustCompile(`^experiments\.([a-zA-Z_][a-zA-Z0-9_]*)$`) +// experimentComparisonRegex matches experiments. followed by a comparison operator and +// a quoted string value, e.g. `experiments.prompt_style == "concise"` or +// `experiments.prompt_style !== "detailed"`. The value must be enclosed in double quotes +// with no embedded quotes. It captures: +// - group 1: the experiment name +// - group 2: the remainder of the expression (operator + quoted value), verbatim +var experimentComparisonRegex = regexp.MustCompile(`^experiments\.([a-zA-Z_][a-zA-Z0-9_]*)([ \t]*(?:!==?|===?)[ \t]*"[^"]*"[ \t]*)$`) + // ExperimentEnvVarName returns the env-var name used for the given experiment. // The name is uppercased; hyphens are converted to underscores; all other characters // that are not A-Z, 0-9, or underscore are dropped (not replaced). @@ -209,17 +217,28 @@ func ExperimentEnvVarName(experimentName string) string { } // transformExperimentsExpression detects expressions of the form "experiments." -// and rewrites them to "steps.pick-experiment.outputs." so that the placeholder -// substitution step reads the value from the pick_experiment step output. -// This is used for ${{ experiments.name }} expressions that appear directly in the prompt body -// (mostly relevant in inline mode; in runtime-import mode the template conditional -// {{#if experiments.name}} path is handled separately via ExperimentExpressionMappings). +// (and the comparison form `experiments. == "value"`) and rewrites them so that the +// placeholder substitution step reads the value from the pick_experiment step output. +// +// Simple form: experiments.name → steps.pick-experiment.outputs.name +// Comparison form: experiments.name == "v" → steps.pick-experiment.outputs.name == "v" +// +// This is used for ${{ experiments.name }} and ${{ experiments.name == "value" }} expressions +// that appear directly in the prompt body (mostly relevant in inline mode; in runtime-import +// mode the {{#if experiments.name == "value"}} conditional is handled by interpolate_prompt.cjs +// step 2.5 which substitutes the variant value directly inside the condition tag). +// +// Without this transformation, the generated env var would contain an invalid GitHub Actions +// expression like `${{ experiments.name == "value" }}` where `experiments` is not a real +// context, causing the expression to always evaluate to false. func transformExperimentsExpression(expr string) string { - m := experimentNameRegex.FindStringSubmatch(expr) - if m == nil { - return expr + if m := experimentNameRegex.FindStringSubmatch(expr); m != nil { + return "steps.pick-experiment.outputs." + m[1] + } + if m := experimentComparisonRegex.FindStringSubmatch(expr); m != nil { + return "steps.pick-experiment.outputs." + m[1] + m[2] } - return "steps.pick-experiment.outputs." + m[1] + return expr } // simpleIdentifierRegex matches simple JavaScript property access chains like diff --git a/pkg/workflow/expression_extraction_test.go b/pkg/workflow/expression_extraction_test.go index a7d2f4d0155..aebb2a2ba1f 100644 --- a/pkg/workflow/expression_extraction_test.go +++ b/pkg/workflow/expression_extraction_test.go @@ -61,6 +61,30 @@ func TestExpressionExtractor_ExtractExpressions(t *testing.T) { wantCount: 1, wantExpressions: []string{"steps.pick-experiment.outputs.caveman"}, }, + { + name: "experiments.name == value comparison form gets transformed to step output", + markdown: `{{#if ${{ experiments.prompt_style == "concise" }} }}foo{{/if}}`, + wantCount: 1, + wantExpressions: []string{`steps.pick-experiment.outputs.prompt_style == "concise"`}, + }, + { + name: "experiments.name === value strict-equality form gets transformed to step output", + markdown: `{{#if ${{ experiments.prompt_style === "concise" }} }}foo{{/if}}`, + wantCount: 1, + wantExpressions: []string{`steps.pick-experiment.outputs.prompt_style === "concise"`}, + }, + { + name: "experiments.name != value inequality form gets transformed to step output", + markdown: `{{#if ${{ experiments.reasoning_depth != "multi_candidate" }} }}foo{{/if}}`, + wantCount: 1, + wantExpressions: []string{`steps.pick-experiment.outputs.reasoning_depth != "multi_candidate"`}, + }, + { + name: "experiments.name !== value strict-inequality form gets transformed to step output", + markdown: `{{#if ${{ experiments.reasoning_depth !== "multi_candidate" }} }}foo{{/if}}`, + wantCount: 1, + wantExpressions: []string{`steps.pick-experiment.outputs.reasoning_depth !== "multi_candidate"`}, + }, { name: "expression with whitespace", markdown: "Value: ${{ github.actor }}",