diff --git a/.github/workflows/daily-model-inventory.lock.yml b/.github/workflows/daily-model-inventory.lock.yml index 138ab35b739..41117939af6 100644 --- a/.github/workflows/daily-model-inventory.lock.yml +++ b/.github/workflows/daily-model-inventory.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"02091f9aef207d6856329f85921dba158a2db7dc2d1014ce3d88d7eaea4beb39","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"36344c8930ef75540d3052f12eb3ecbe31dbcb94231f8d9c82551bd120ce3ce8","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GEMINI_API_KEY","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_ENDPOINT","GH_AW_OTEL_HEADERS","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.38"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.38"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.38"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.6"},{"image":"ghcr.io/github/github-mcp-server:v1.0.3","digest":"sha256:2ac27ef03461ef2b877031b838a7d1fd7f12b12d4ace7796d8cad91446d55959","pinned_image":"ghcr.io/github/github-mcp-server:v1.0.3@sha256:2ac27ef03461ef2b877031b838a7d1fd7f12b12d4ace7796d8cad91446d55959"},{"image":"node:lts-alpine","digest":"sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f","pinned_image":"node:lts-alpine@sha256:d1b3b4da11eefd5941e7f0b9cf17783fc99d9c6fc34884a665f40a06dbdfc94f"}]} # ___ _ _ # / _ \ | | (_) @@ -197,21 +197,21 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_d4e4560568e99794_EOF' + cat << 'GH_AW_PROMPT_0d7b5547df10fa50_EOF' - GH_AW_PROMPT_d4e4560568e99794_EOF + GH_AW_PROMPT_0d7b5547df10fa50_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/playwright_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_d4e4560568e99794_EOF' + cat << 'GH_AW_PROMPT_0d7b5547df10fa50_EOF' Tools: create_issue, missing_tool, missing_data, noop - GH_AW_PROMPT_d4e4560568e99794_EOF + GH_AW_PROMPT_0d7b5547df10fa50_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_d4e4560568e99794_EOF' + cat << 'GH_AW_PROMPT_0d7b5547df10fa50_EOF' The following GitHub context information is available for this workflow: {{#if __GH_AW_GITHUB_ACTOR__ }} @@ -240,15 +240,15 @@ jobs: {{/if}} - GH_AW_PROMPT_d4e4560568e99794_EOF + GH_AW_PROMPT_0d7b5547df10fa50_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_d4e4560568e99794_EOF' + cat << 'GH_AW_PROMPT_0d7b5547df10fa50_EOF' {{#runtime-import .github/workflows/shared/otel.md}} {{#runtime-import .github/workflows/shared/observability-otlp.md}} {{#runtime-import .github/workflows/shared/noop-reminder.md}} {{#runtime-import .github/workflows/daily-model-inventory.md}} - GH_AW_PROMPT_d4e4560568e99794_EOF + GH_AW_PROMPT_0d7b5547df10fa50_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -471,9 +471,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_ff5f5cdf0faea0b8_EOF' + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_e528017d11c3457d_EOF' {"create_issue":{"close_older_issues":true,"expires":168,"labels":["automation","models"],"max":1,"title_prefix":"[model-inventory] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_ff5f5cdf0faea0b8_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_e528017d11c3457d_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -675,7 +675,7 @@ jobs: mkdir -p /home/runner/.copilot GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_9505bace89d8359e_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_454332c1637f4835_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { @@ -722,7 +722,7 @@ jobs: } } } - GH_AW_MCP_CONFIG_9505bace89d8359e_EOF + GH_AW_MCP_CONFIG_454332c1637f4835_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true @@ -752,7 +752,7 @@ jobs: # --allow-tool shell(cat /tmp/gh-aw/model-inventory/artifacts/copilot-billing-multipliers/multipliers.json) # --allow-tool shell(cat /tmp/gh-aw/model-inventory/inventory.json) # --allow-tool shell(cat pkg/cli/data/model_multipliers.json) - # --allow-tool shell(cat pkg/workflow/model_aliases.go) + # --allow-tool shell(cat pkg/workflow/data/model_aliases.json) # --allow-tool shell(cat) # --allow-tool shell(date) # --allow-tool shell(echo) @@ -784,7 +784,7 @@ jobs: printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.38/awf-config.schema.json","network":{"allowDomains":["api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","cdn.playwright.dev","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","playwright.download.prss.microsoft.com","ppa.launchpad.net","raw.githubusercontent.com","registry.npmjs.org","s.symcb.com","s.symcd.com","security.ubuntu.com","telemetry.enterprise.githubcopilot.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com"]},"apiProxy":{"enabled":true},"container":{"imageTag":"0.25.38"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" && cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json # shellcheck disable=SC1003 sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ - -- /bin/bash -c 'export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 4 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || echo node)"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(cat /tmp/gh-aw/model-inventory/artifacts/copilot-billing-multipliers/multipliers.json)'\'' --allow-tool '\''shell(cat /tmp/gh-aw/model-inventory/inventory.json)'\'' --allow-tool '\''shell(cat pkg/cli/data/model_multipliers.json)'\'' --allow-tool '\''shell(cat pkg/workflow/model_aliases.go)'\'' --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(find /tmp/gh-aw/model-inventory -type f)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(jq . /tmp/gh-aw/model-inventory/artifacts/*/models.json)'\'' --allow-tool '\''shell(jq . /tmp/gh-aw/model-inventory/artifacts/*/raw.json)'\'' --allow-tool '\''shell(jq . /tmp/gh-aw/model-inventory/artifacts/copilot-billing-multipliers/multipliers.json)'\'' --allow-tool '\''shell(jq . /tmp/gh-aw/model-inventory/inventory.json)'\'' --allow-tool '\''shell(jq)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(playwright-cli:*)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + -- /bin/bash -c 'export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && export PATH="$(find /opt/hostedtoolcache /home/runner/work/_tool -maxdepth 4 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || echo node)"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(cat /tmp/gh-aw/model-inventory/artifacts/copilot-billing-multipliers/multipliers.json)'\'' --allow-tool '\''shell(cat /tmp/gh-aw/model-inventory/inventory.json)'\'' --allow-tool '\''shell(cat pkg/cli/data/model_multipliers.json)'\'' --allow-tool '\''shell(cat pkg/workflow/data/model_aliases.json)'\'' --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(find /tmp/gh-aw/model-inventory -type f)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(jq . /tmp/gh-aw/model-inventory/artifacts/*/models.json)'\'' --allow-tool '\''shell(jq . /tmp/gh-aw/model-inventory/artifacts/*/raw.json)'\'' --allow-tool '\''shell(jq . /tmp/gh-aw/model-inventory/artifacts/copilot-billing-multipliers/multipliers.json)'\'' --allow-tool '\''shell(jq . /tmp/gh-aw/model-inventory/inventory.json)'\'' --allow-tool '\''shell(jq)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(playwright-cli:*)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_API_KEY: dummy-byok-key-for-offline-mode diff --git a/.github/workflows/daily-model-inventory.md b/.github/workflows/daily-model-inventory.md index 6116d3d935f..8e305f3cb69 100644 --- a/.github/workflows/daily-model-inventory.md +++ b/.github/workflows/daily-model-inventory.md @@ -362,7 +362,7 @@ tools: - "jq . /tmp/gh-aw/model-inventory/artifacts/copilot-billing-multipliers/multipliers.json" - "jq '.models[]' /tmp/gh-aw/model-inventory/artifacts/copilot-billing-multipliers/multipliers.json" - "find /tmp/gh-aw/model-inventory -type f" - - "cat pkg/workflow/model_aliases.go" + - "cat pkg/workflow/data/model_aliases.json" - "cat pkg/cli/data/model_multipliers.json" github: toolsets: [default] @@ -384,7 +384,7 @@ imports: You are an AI model catalog analyst for `${{ github.repository }}`. Your task is to analyze the current model inventories from all configured AI providers and -determine whether the built-in model alias mapping in `pkg/workflow/model_aliases.go` needs +determine whether the built-in model alias mapping in `pkg/workflow/data/model_aliases.json` needs updating. ## Inputs @@ -418,7 +418,7 @@ empty `models` array. Skip providers with errors or empty model lists. ## Built-in Alias Reference -Read `pkg/workflow/model_aliases.go` to understand the current alias definitions. The current +Read `pkg/workflow/data/model_aliases.json` to understand the current alias definitions. The current built-in aliases are: | Alias | Resolves to | @@ -507,7 +507,7 @@ Produce a consolidated multiplier gap table listing: ### Step 4: Identify New or Updated Model Families -Compare the live model list against the current aliases in `pkg/workflow/model_aliases.go`. +Compare the live model list against the current aliases in `pkg/workflow/data/model_aliases.json`. Look for: 1. **New model generations** — e.g. a new `claude-sonnet-5` or `gpt-6` that is not covered by @@ -523,14 +523,16 @@ Look for: ### Step 5: Propose Alias Mapping Updates -For each finding from Step 4, produce a concrete YAML snippet showing the proposed new or updated -alias entry in the `models:` frontmatter format. Use the alias pattern syntax: +For each finding from Step 4, produce a concrete JSON snippet showing the proposed new or updated +alias entry in the `aliases` object in `pkg/workflow/data/model_aliases.json`. Use the alias pattern syntax: -```yaml -models: - new-alias: - - "copilot/vendor-model-id*" - - "vendor/vendor-model-id*" +```json +{ + "new-alias": [ + "copilot/vendor-model-id*", + "vendor/vendor-model-id*" + ] +} ``` Focus on aliases that provide genuine value to workflow authors. Prioritize: @@ -593,7 +595,7 @@ List model IDs that appear in `model_multipliers.json` but are absent from all l For each change, explain: 1. **What**: The alias name and new/updated patterns 2. **Why**: Which live model(s) prompted this change -3. **Syntax**: YAML snippet ready to copy into `pkg/workflow/model_aliases.go` +3. **Syntax**: JSON snippet showing the new or updated entry for the `aliases` object in `pkg/workflow/data/model_aliases.json`
Full Model Lists by Provider diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index f26106a16b6..7e868faa274 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -331,6 +331,7 @@ export default defineConfig({ { label: 'Cost Management', link: '/reference/cost-management/' }, { label: 'Cost Management (Rate Limiting)', link: '/reference/rate-limiting-controls/' }, { label: 'Cost Management (Effective Tokens)', link: '/reference/effective-tokens-specification/' }, + { label: 'Cost Management (Model Tables)', link: '/reference/model-tables/' }, { label: 'Environment Variables', link: '/reference/environment-variables/' }, { label: 'FAQ', link: '/reference/faq/' }, { label: 'Footers', link: '/reference/footers/' }, diff --git a/docs/package.json b/docs/package.json index 1c3e3bda4c4..9026986aa0f 100644 --- a/docs/package.json +++ b/docs/package.json @@ -5,13 +5,14 @@ "scripts": { "dev": "astro dev", "start": "astro dev", - "prebuild": "npm run generate-agent-factory && npm run build:slides", + "prebuild": "npm run generate-agent-factory && npm run generate-model-tables && npm run build:slides", "build": "astro build", "build:slides": "mkdir -p public/js && cp node_modules/mermaid/dist/mermaid.min.js public/js/ && npx @marp-team/marp-cli slides/index.md --html --allow-local-files --theme slides/gh-aw-theme.css -o public/slides/index.html && mkdir -p public/slides && cp slides/*.pdf public/slides/", "preview": "astro preview", "astro": "astro", "validate-links": "astro build", "generate-agent-factory": "cd .. && node scripts/generate-agent-factory.js", + "generate-model-tables": "cd .. && node scripts/generate-model-tables.js", "test": "playwright test", "test:ui": "playwright test --ui", "test:debug": "playwright test --debug" diff --git a/docs/src/content/docs/reference/model-tables.md b/docs/src/content/docs/reference/model-tables.md new file mode 100644 index 00000000000..94da8985824 --- /dev/null +++ b/docs/src/content/docs/reference/model-tables.md @@ -0,0 +1,154 @@ +--- +title: Model Aliases & Multipliers +description: Reference tables for the built-in model alias map and per-model Effective Token multipliers used by GitHub Agentic Workflows. +sidebar: + order: 297 +--- + +This page lists the built-in model aliases and the per-model Effective Token (ET) multipliers used by GitHub Agentic Workflows. + +> [!CAUTION] +> The multiplier values shown on this page are **approximations**. They are used solely for the purpose of normalizing token usage across models into a single comparable metric (Effective Tokens) and do **not** represent precise cost ratios. Values may be inaccurate for specific model versions and may become out of date as providers update their offerings. Do not use these numbers for billing or financial calculations. + +## Model Aliases + +Model aliases let you write `engine: copilot` with a human-friendly model name such as `sonnet` or `mini`, and gh-aw resolves it to the best available concrete model at compile time. Each alias holds an ordered list of patterns; the first pattern that matches an available model wins. + +For details on the alias syntax, fallback resolution algorithm, and how to define your own aliases in workflow frontmatter, see the [Model Alias Format Specification](/gh-aw/reference/model-alias-specification/). + +### Vendor Aliases + +Vendor aliases map a short name to one or more provider-scoped glob patterns. The Copilot gateway is always tried first. + +| Alias | Fallback patterns (tried in order) | +|-------|-------------------------------------| +| `sonnet` | `copilot/*sonnet*`, `anthropic/*sonnet*` | +| `haiku` | `copilot/*haiku*`, `anthropic/*haiku*` | +| `opus` | `copilot/*opus*`, `anthropic/*opus*` | +| `gpt-4.1` | `copilot/gpt-4.1*`, `openai/gpt-4.1*` | +| `gpt-5` | `copilot/gpt-5*`, `openai/gpt-5*` | +| `gpt-5-mini` | `copilot/gpt-5*mini*`, `openai/gpt-5*mini*` | +| `gpt-5-nano` | `copilot/gpt-5*nano*`, `openai/gpt-5*nano*` | +| `gpt-5-codex` | `copilot/gpt-5*codex*`, `openai/gpt-5*codex*` | +| `gpt-5-pro` | `copilot/gpt-5*pro*`, `openai/gpt-5*pro*` | +| `reasoning` | `copilot/o1*`, `copilot/o3*`, `copilot/o4*`, `openai/o1*`, `openai/o3*`, `openai/o4*` | +| `gemini-flash` | `copilot/gemini-*flash*`, `google/gemini-*flash*` | +| `gemini-pro` | `copilot/gemini-*pro*`, `google/gemini-*pro*` | +| `deep-research` | `copilot/deep-research*`, `google/deep-research*` | + +### Meta-Aliases + +Meta-aliases reference other aliases by name. They are resolved recursively until a concrete pattern is reached. + +| Meta-alias | Expands to | +|------------|------------| +| `small` | `mini` | +| `mini` | `haiku` → `gpt-5-mini` → `gpt-5-nano` → `gemini-flash` | +| `large` | `sonnet` → `gpt-5-pro` → `gpt-5` → `gemini-pro` | +| `auto` | `large` | + +## Model Multipliers + +Effective Token multipliers scale the weighted token total for each model relative to the reference model (`claude-sonnet-4.5`, multiplier = 1.0). A multiplier of 5.0 means that a run on that model counts as five times as many Effective Tokens as the same run on the reference model. + +See the [Effective Tokens Specification](/gh-aw/reference/effective-tokens-specification/) for the full formula. + +### Token Class Weights + +Before per-model multipliers are applied, raw token counts are weighted by token class: + +| Token class | Default weight | +|-------------|---------------| +| Input | 1 | +| Cached Input | 0.1 | +| Output | 4 | +| Reasoning | 4 | +| Cache Write | 1 | + +### Per-Model Multipliers + +### Anthropic + +| Model | Multiplier | +|-------|-----------| +| `claude-haiku-4-5` | 0.1 | +| `claude-haiku-4.5` | 0.1 | +| `claude-3-5-haiku` | 0.1 | +| `claude-3-haiku` | 0.1 | +| `claude-sonnet-4` | 1 | +| `claude-sonnet-4-5` | 1 | +| `claude-sonnet-4.5` | 1 | +| `claude-sonnet-4.6` | 1 | +| `claude-3-5-sonnet` | 1 | +| `claude-3-7-sonnet` | 1 | +| `claude-3-sonnet` | 1 | +| `claude-opus-4` | 5 | +| `claude-opus-4-1` | 5 | +| `claude-opus-4-5` | 5 | +| `claude-opus-4-6` | 5 | +| `claude-opus-4-7` | 5 | +| `claude-opus-4.5` | 5 | +| `claude-opus-4.6` | 5 | +| `claude-3-5-opus` | 5 | +| `claude-3-opus` | 5 | + +### OpenAI + +| Model | Multiplier | +|-------|-----------| +| `gpt-4o` | 1 | +| `gpt-4o-mini` | 0.1 | +| `gpt-4.1` | 1 | +| `gpt-4.1-2025-04-14` | 1 | +| `gpt-41-copilot` | 1 | +| `gpt-4.1-mini` | 0.1 | +| `gpt-4.1-nano` | 0.05 | +| `gpt-4-turbo` | 1 | +| `gpt-4` | 1 | +| `gpt-5` | 1 | +| `gpt-5-mini` | 0.33 | +| `gpt-5-nano` | 0.05 | +| `gpt-5-pro` | 2 | +| `gpt-5.1` | 1 | +| `gpt-5.1-codex` | 1 | +| `gpt-5.1-codex-mini` | 0.33 | +| `gpt-5.1-codex-max` | 1 | +| `gpt-5.2` | 1 | +| `gpt-5.2-codex` | 1 | +| `gpt-5.2-pro` | 2 | +| `gpt-5.3-codex` | 1 | +| `gpt-5.4` | 1 | +| `gpt-5.4-mini` | 0.33 | +| `gpt-5.4-pro` | 2 | +| `gpt-5.5` | 1 | +| `gpt-5.5-pro` | 2 | + +### OpenAI Reasoning + +| Model | Multiplier | +|-------|-----------| +| `o1` | 3 | +| `o1-mini` | 0.5 | +| `o1-pro` | 10 | +| `o3` | 3 | +| `o3-mini` | 0.5 | +| `o3-pro` | 10 | +| `o4-mini` | 0.5 | + +### Google + +| Model | Multiplier | +|-------|-----------| +| `gemini-2.5-pro` | 1 | +| `gemini-2.5-flash` | 0.2 | +| `gemini-2.5-flash-image` | 0.2 | +| `gemini-2.5-flash-lite` | 0.1 | +| `gemini-2.0-flash` | 0.1 | +| `gemini-1.5-pro` | 1 | +| `gemini-1.5-flash` | 0.1 | +| `gemini-3-flash-preview` | 0.33 | +| `gemini-3-pro-preview` | 6 | +| `gemini-3-pro-image-preview` | 6 | +| `gemini-3.1-pro-preview` | 6 | +| `gemini-3.1-flash-lite-preview` | 0.1 | +| `gemini-3.1-flash-image-preview` | 0.33 | diff --git a/pkg/workflow/data/model_aliases.json b/pkg/workflow/data/model_aliases.json new file mode 100644 index 00000000000..20c9b0f021d --- /dev/null +++ b/pkg/workflow/data/model_aliases.json @@ -0,0 +1,23 @@ +{ + "version": "1", + "description": "Built-in model alias map for GitHub Agentic Workflows. Each key is an alias name; the value is an ordered list of model patterns tried in sequence until one resolves.", + "aliases": { + "sonnet": ["copilot/*sonnet*", "anthropic/*sonnet*"], + "haiku": ["copilot/*haiku*", "anthropic/*haiku*"], + "opus": ["copilot/*opus*", "anthropic/*opus*"], + "gpt-4.1": ["copilot/gpt-4.1*", "openai/gpt-4.1*"], + "gpt-5": ["copilot/gpt-5*", "openai/gpt-5*"], + "gpt-5-mini": ["copilot/gpt-5*mini*", "openai/gpt-5*mini*"], + "gpt-5-nano": ["copilot/gpt-5*nano*", "openai/gpt-5*nano*"], + "gpt-5-codex": ["copilot/gpt-5*codex*", "openai/gpt-5*codex*"], + "gpt-5-pro": ["copilot/gpt-5*pro*", "openai/gpt-5*pro*"], + "reasoning": ["copilot/o1*", "copilot/o3*", "copilot/o4*", "openai/o1*", "openai/o3*", "openai/o4*"], + "gemini-flash": ["copilot/gemini-*flash*", "google/gemini-*flash*"], + "gemini-pro": ["copilot/gemini-*pro*", "google/gemini-*pro*"], + "deep-research": ["copilot/deep-research*", "google/deep-research*"], + "small": ["mini"], + "mini": ["haiku", "gpt-5-mini", "gpt-5-nano", "gemini-flash"], + "large": ["sonnet", "gpt-5-pro", "gpt-5", "gemini-pro"], + "auto": ["large"] + } +} diff --git a/pkg/workflow/model_aliases.go b/pkg/workflow/model_aliases.go index 183f0865e70..02b50d89b7d 100644 --- a/pkg/workflow/model_aliases.go +++ b/pkg/workflow/model_aliases.go @@ -20,12 +20,16 @@ // # Builtin Aliases // // gh-aw ships a set of builtin model aliases that cover the major model families. +// The alias data is defined in data/model_aliases.json (embedded at compile time). // Frontmatter-defined aliases are merged on top of the builtins, allowing workflows // to extend or override the defaults without replacing the entire mapping. package workflow import ( + _ "embed" + "encoding/json" + "fmt" "maps" "github.com/github/gh-aw/pkg/logger" @@ -33,10 +37,19 @@ import ( var modelAliasesLog = logger.New("workflow:model_aliases") +//go:embed data/model_aliases.json +var builtinModelAliasesJSON []byte + +// builtinModelAliasesFile mirrors the JSON structure of data/model_aliases.json. +type builtinModelAliasesFile struct { + Aliases map[string][]string `json:"aliases"` +} + // BuiltinModelAliases returns the built-in model alias map that covers the main // model families supported by gh-aw. The returned map is a freshly allocated // copy so callers may freely modify it. // +// The alias data is loaded from data/model_aliases.json (embedded at compile time). // Vendor aliases (patterns use * as a glob wildcard, prefer copilot gateway first): // - "sonnet" → Anthropic Sonnet family // - "haiku" → Anthropic Haiku family @@ -57,91 +70,14 @@ var modelAliasesLog = logger.New("workflow:model_aliases") // - "large" → sonnet, gpt-5-pro, gpt-5, gemini-pro // - "auto" → large (convenience alias for the default capable tier) func BuiltinModelAliases() map[string][]string { - return map[string][]string{ - // ── Anthropic ──────────────────────────────────────────────────────── - "sonnet": { - "copilot/*sonnet*", - "anthropic/*sonnet*", - }, - "haiku": { - "copilot/*haiku*", - "anthropic/*haiku*", - }, - "opus": { - "copilot/*opus*", - "anthropic/*opus*", - }, - // ── OpenAI ─────────────────────────────────────────────────────────── - "gpt-4.1": { - "copilot/gpt-4.1*", - "openai/gpt-4.1*", - }, - "gpt-5": { - "copilot/gpt-5*", - "openai/gpt-5*", - }, - "gpt-5-mini": { - "copilot/gpt-5*mini*", - "openai/gpt-5*mini*", - }, - "gpt-5-nano": { - "copilot/gpt-5*nano*", - "openai/gpt-5*nano*", - }, - "gpt-5-codex": { - "copilot/gpt-5*codex*", - "openai/gpt-5*codex*", - }, - "gpt-5-pro": { - "copilot/gpt-5*pro*", - "openai/gpt-5*pro*", - }, - "reasoning": { - "copilot/o1*", - "copilot/o3*", - "copilot/o4*", - "openai/o1*", - "openai/o3*", - "openai/o4*", - }, - // ── Google ─────────────────────────────────────────────────────────── - "gemini-flash": { - "copilot/gemini-*flash*", - "google/gemini-*flash*", - }, - "gemini-pro": { - "copilot/gemini-*pro*", - "google/gemini-*pro*", - }, - "deep-research": { - "copilot/deep-research*", - "google/deep-research*", - }, - // ── Meta-aliases ───────────────────────────────────────────────────── - // These reference other aliases; AWF resolves them recursively. - // "small" — same as "mini" (convenience alias for lightweight/fast models). - // "mini" — lightweight/fast models across all supported vendors. - // "large" — full-capability models across all supported vendors. - // "auto" — convenience alias that resolves to the "large" tier. - "small": { - "mini", - }, - "mini": { - "haiku", - "gpt-5-mini", - "gpt-5-nano", - "gemini-flash", - }, - "large": { - "sonnet", - "gpt-5-pro", - "gpt-5", - "gemini-pro", - }, - "auto": { - "large", - }, + var data builtinModelAliasesFile + if err := json.Unmarshal(builtinModelAliasesJSON, &data); err != nil { + panic(fmt.Sprintf("workflow: failed to parse embedded model_aliases.json: %v (try 'make build' to rebuild with the latest data)", err)) } + // Return a fresh copy so callers may freely modify it. + result := make(map[string][]string, len(data.Aliases)) + maps.Copy(result, data.Aliases) + return result } // MergeImportedModelAliases builds the final model alias map from three layers, diff --git a/scripts/generate-model-tables.js b/scripts/generate-model-tables.js new file mode 100644 index 00000000000..47cfd8d01de --- /dev/null +++ b/scripts/generate-model-tables.js @@ -0,0 +1,262 @@ +#!/usr/bin/env node +/** + * Model Tables Documentation Generator + * + * Reads the built-in model alias map and model multipliers from JSON data files + * and generates an intuitively readable reference page with markdown tables. + * + * Usage: + * node scripts/generate-model-tables.js + * + * Inputs: + * pkg/cli/data/model_aliases.json – Built-in alias → pattern mappings + * pkg/cli/data/model_multipliers.json – Per-model Effective Token multipliers + * + * Output: + * docs/src/content/docs/reference/model-tables.md + */ + +import fs from "fs"; +import path from "path"; +import { fileURLToPath } from "url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const ROOT = path.resolve(__dirname, ".."); +const ALIASES_PATH = path.join(ROOT, "pkg/workflow/data/model_aliases.json"); +const MULTIPLIERS_PATH = path.join(ROOT, "pkg/cli/data/model_multipliers.json"); +const OUTPUT_PATH = path.join(ROOT, "docs/src/content/docs/reference/model-tables.md"); + +// --------------------------------------------------------------------------- +// Load data +// --------------------------------------------------------------------------- + +/** + * Read and parse a JSON file, with a clear error message on failure. + * @param {string} filePath + * @returns {any} + */ +function readJSON(filePath) { + let raw; + try { + raw = fs.readFileSync(filePath, "utf-8"); + } catch (err) { + console.error(`Error: could not read ${filePath}: ${err.message}`); + process.exit(1); + } + try { + return JSON.parse(raw); + } catch (err) { + console.error(`Error: invalid JSON in ${filePath}: ${err.message}`); + process.exit(1); + } +} + +const aliasesData = readJSON(ALIASES_PATH); +const multipliersData = readJSON(MULTIPLIERS_PATH); + +const aliases = aliasesData.aliases; +const multipliers = multipliersData.multipliers; +const tokenClassWeights = multipliersData.token_class_weights; +const referenceModel = multipliersData.reference_model; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Group aliases into "vendor" aliases (patterns using glob wildcards) and + * "meta" aliases (patterns that reference other aliases by name, with no slash). + */ +function classifyAliases(aliasMap) { + const vendor = []; + const meta = []; + for (const [alias, patterns] of Object.entries(aliasMap)) { + const isMetaOnly = patterns.every(p => !p.includes("/")); + if (isMetaOnly) { + meta.push({ alias, resolves: patterns }); + } else { + vendor.push({ alias, patterns }); + } + } + return { vendor, meta }; +} + +/** + * Group multipliers into provider sections based on model name prefix. + */ +function groupMultipliers(mults) { + const groups = { + Anthropic: [], + OpenAI: [], + "OpenAI Reasoning": [], + Google: [], + Other: [], + }; + + for (const [model, value] of Object.entries(mults)) { + if (model.startsWith("claude-")) { + groups["Anthropic"].push({ model, value }); + } else if (/^o[0-9]/.test(model)) { + groups["OpenAI Reasoning"].push({ model, value }); + } else if (model.startsWith("gpt-")) { + groups["OpenAI"].push({ model, value }); + } else if (model.startsWith("gemini-")) { + groups["Google"].push({ model, value }); + } else { + groups["Other"].push({ model, value }); + } + } + + // Remove empty groups + return Object.fromEntries(Object.entries(groups).filter(([, rows]) => rows.length > 0)); +} + +// --------------------------------------------------------------------------- +// Markdown generators +// --------------------------------------------------------------------------- + +function generateAliasTable(vendorAliases) { + const lines = []; + lines.push("| Alias | Fallback patterns (tried in order) |"); + lines.push("|-------|-------------------------------------|"); + for (const { alias, patterns } of vendorAliases) { + const formattedPatterns = patterns.map(p => `\`${p}\``).join(", "); + lines.push(`| \`${alias}\` | ${formattedPatterns} |`); + } + return lines.join("\n"); +} + +function generateMetaAliasTable(metaAliases) { + const lines = []; + lines.push("| Meta-alias | Expands to |"); + lines.push("|------------|------------|"); + for (const { alias, resolves } of metaAliases) { + const formattedResolves = resolves.map(r => `\`${r}\``).join(" → "); + lines.push(`| \`${alias}\` | ${formattedResolves} |`); + } + return lines.join("\n"); +} + +function generateMultiplierSection(groupName, rows) { + const lines = []; + lines.push(`### ${groupName}`); + lines.push(""); + lines.push("| Model | Multiplier |"); + lines.push("|-------|-----------|"); + for (const { model, value } of rows) { + lines.push(`| \`${model}\` | ${value} |`); + } + return lines.join("\n"); +} + +function generateTokenWeightsTable(weights) { + const lines = []; + lines.push("| Token class | Default weight |"); + lines.push("|-------------|---------------|"); + for (const [cls, weight] of Object.entries(weights)) { + const label = cls.replace(/_/g, " ").replace(/\b\w/g, c => c.toUpperCase()); + lines.push(`| ${label} | ${weight} |`); + } + return lines.join("\n"); +} + +// --------------------------------------------------------------------------- +// Build the full document +// --------------------------------------------------------------------------- + +function generateMarkdown() { + const { vendor, meta } = classifyAliases(aliases); + const multiplierGroups = groupMultipliers(multipliers); + + const lines = []; + + // Frontmatter + lines.push("---"); + lines.push("title: Model Aliases & Multipliers"); + lines.push("description: Reference tables for the built-in model alias map and per-model Effective Token multipliers used by GitHub Agentic Workflows."); + lines.push("sidebar:"); + lines.push(" order: 297"); + lines.push("---"); + lines.push(""); + + lines.push("This page lists the built-in model aliases and the per-model Effective Token (ET) multipliers used by GitHub Agentic Workflows."); + lines.push(""); + + // Approximation callout + lines.push("> [!CAUTION]"); + lines.push( + "> The multiplier values shown on this page are **approximations**. They are used solely for the purpose of normalizing token usage across models into a single comparable metric (Effective Tokens) and do **not** represent precise cost ratios. Values may be inaccurate for specific model versions and may become out of date as providers update their offerings. Do not use these numbers for billing or financial calculations." + ); + lines.push(""); + + // ------------------------------------------------------------------------- + // Model Aliases + // ------------------------------------------------------------------------- + lines.push("## Model Aliases"); + lines.push(""); + lines.push( + "Model aliases let you write `engine: copilot` with a human-friendly model name such as `sonnet` or `mini`, and gh-aw resolves it to the best available concrete model at compile time. Each alias holds an ordered list of patterns; the first pattern that matches an available model wins." + ); + lines.push(""); + lines.push("For details on the alias syntax, fallback resolution algorithm, and how to define your own aliases in workflow frontmatter, see the [Model Alias Format Specification](/gh-aw/reference/model-alias-specification/)."); + lines.push(""); + + lines.push("### Vendor Aliases"); + lines.push(""); + lines.push("Vendor aliases map a short name to one or more provider-scoped glob patterns. The Copilot gateway is always tried first."); + lines.push(""); + lines.push(generateAliasTable(vendor)); + lines.push(""); + + lines.push("### Meta-Aliases"); + lines.push(""); + lines.push("Meta-aliases reference other aliases by name. They are resolved recursively until a concrete pattern is reached."); + lines.push(""); + lines.push(generateMetaAliasTable(meta)); + lines.push(""); + + // ------------------------------------------------------------------------- + // Model Multipliers + // ------------------------------------------------------------------------- + lines.push("## Model Multipliers"); + lines.push(""); + lines.push( + `Effective Token multipliers scale the weighted token total for each model relative to the reference model (\`${referenceModel}\`, multiplier = 1.0). A multiplier of 5.0 means that a run on that model counts as five times as many Effective Tokens as the same run on the reference model.` + ); + lines.push(""); + lines.push("See the [Effective Tokens Specification](/gh-aw/reference/effective-tokens-specification/) for the full formula."); + lines.push(""); + + lines.push("### Token Class Weights"); + lines.push(""); + lines.push("Before per-model multipliers are applied, raw token counts are weighted by token class:"); + lines.push(""); + lines.push(generateTokenWeightsTable(tokenClassWeights)); + lines.push(""); + + lines.push("### Per-Model Multipliers"); + lines.push(""); + for (const [groupName, rows] of Object.entries(multiplierGroups)) { + lines.push(generateMultiplierSection(groupName, rows)); + lines.push(""); + } + + return lines.join("\n"); +} + +// --------------------------------------------------------------------------- +// Write output +// --------------------------------------------------------------------------- +console.log("Generating model tables documentation..."); + +const markdown = generateMarkdown(); + +const outputDir = path.dirname(OUTPUT_PATH); +if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); +} + +fs.writeFileSync(OUTPUT_PATH, markdown, "utf-8"); +console.log(`✓ Generated: ${OUTPUT_PATH}`);