Publish Channel and Site #16187
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish Channel and Site | |
| concurrency: | |
| group: crawl-pipeline | |
| cancel-in-progress: false | |
| on: | |
| workflow_run: | |
| workflows: ["Crawl Registry"] | |
| types: | |
| - completed | |
| workflow_dispatch: | |
| push: | |
| tags: | |
| - '*' | |
| permissions: | |
| contents: write | |
| jobs: | |
| build-st4-channel: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'workflow_run' || github.event_name == 'workflow_dispatch' | |
| outputs: | |
| update_needed: ${{ steps.check.outputs.update_needed }} | |
| env: | |
| RELEASE_TAG: the-channel | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: Install uv | |
| run: | | |
| pip install uv | |
| uv sync | |
| - name: Generate new channel.json | |
| id: generate | |
| run: | | |
| PYTHONUNBUFFERED=1 uv run -m scripts.compress_channel \ | |
| 2>&1 | tee channel.log | |
| # Verify the file was created | |
| if [ ! -f "channel.json" ]; then | |
| echo "Error: channel.json was not generated" | |
| exit 1 | |
| fi | |
| # Calculate hash of new file | |
| NEW_HASH=$(sha256sum channel.json | cut -d ' ' -f 1) | |
| echo "new_hash=$NEW_HASH" >> $GITHUB_OUTPUT | |
| echo "New channel.json hash: $NEW_HASH" | |
| - name: Try to download existing channel.json | |
| id: download | |
| continue-on-error: true | |
| run: | | |
| if gh release download ${{ env.RELEASE_TAG }} --pattern "channel.json" --output existing-channel.json; then | |
| EXISTING_HASH=$(sha256sum existing-channel.json | cut -d ' ' -f 1) | |
| echo "existing_hash=$EXISTING_HASH" >> $GITHUB_OUTPUT | |
| echo "Existing channel.json hash: $EXISTING_HASH" | |
| else | |
| echo "No existing channel.json found or download failed" | |
| echo "existing_hash=none" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Determine if update is needed | |
| id: check | |
| run: | | |
| # If first run or hashes differ, update is needed | |
| if [ "${{ steps.download.outputs.existing_hash }}" != "${{ steps.generate.outputs.new_hash }}" ]; then | |
| echo "update_needed=true" >> $GITHUB_OUTPUT | |
| echo "Update needed: Content has changed or first run" | |
| else | |
| echo "update_needed=false" >> $GITHUB_OUTPUT | |
| echo "No update needed: Content is unchanged" | |
| fi | |
| - name: Update release and upload asset if needed | |
| if: steps.check.outputs.update_needed == 'true' | |
| run: | | |
| # Create or update the release | |
| gh release view ${{ env.RELEASE_TAG }} || \ | |
| gh release create ${{ env.RELEASE_TAG }} \ | |
| --title "Channel Asset" \ | |
| --notes "channel.json" | |
| echo "Uploading channel.json file..." | |
| gh release upload ${{ env.RELEASE_TAG }} channel.json --clobber | |
| # Format release notes | |
| HASH=${{ steps.generate.outputs.new_hash }} | |
| DATE=$(date -u +"%B %d, %Y, %H:%M UTC") | |
| REPO_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| # Build new notes | |
| { | |
| echo "channel.json" | |
| echo "sha256: $HASH" | |
| echo "date: $DATE" | |
| echo "([logs]($REPO_URL))" | |
| echo "" | |
| echo "---" | |
| cat channel.log | |
| } > notes.txt | |
| echo "Updating release notes..." | |
| gh release edit ${{ env.RELEASE_TAG }} --notes-file notes.txt | |
| - name: Output result | |
| run: | | |
| if [ "${{ steps.check.outputs.update_needed }}" = "true" ]; then | |
| echo "✅ Channel updated with new content" | |
| else | |
| echo "ℹ️ No changes detected in channel.json, skipped update" | |
| fi | |
| - name: Show diff (if update and not first run) | |
| if: steps.check.outputs.update_needed == 'true' && steps.download.outputs.existing_hash != 'none' | |
| run: | | |
| echo "📄 Showing diff between existing-channel.json and channel.json" | |
| python -m json.tool existing-channel.json > old.json | |
| python -m json.tool channel.json > new.json | |
| diff -u old.json new.json || true | |
| build-st3-channel: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'workflow_run' || github.event_name == 'workflow_dispatch' | |
| env: | |
| RELEASE_TAG: the-st3-channel | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: Install uv | |
| run: | | |
| pip install uv | |
| uv sync | |
| - name: Generate new channel_st3.json | |
| id: generate | |
| run: | | |
| PYTHONUNBUFFERED=1 uv run -m scripts.compress_channel --legacy -o channel_st3.json \ | |
| 2>&1 | tee channel.log | |
| # Verify the file was created | |
| if [ ! -f "channel_st3.json" ]; then | |
| echo "Error: channel_st3.json was not generated" | |
| exit 1 | |
| fi | |
| # Calculate hash of new file | |
| NEW_HASH=$(sha256sum channel_st3.json | cut -d ' ' -f 1) | |
| echo "new_hash=$NEW_HASH" >> $GITHUB_OUTPUT | |
| echo "New channel_st3.json hash: $NEW_HASH" | |
| - name: Try to download existing channel_st3.json | |
| id: download | |
| continue-on-error: true | |
| run: | | |
| if gh release download ${{ env.RELEASE_TAG }} --pattern "channel_st3.json" --output existing-channel_st3.json; then | |
| EXISTING_HASH=$(sha256sum existing-channel_st3.json | cut -d ' ' -f 1) | |
| echo "existing_hash=$EXISTING_HASH" >> $GITHUB_OUTPUT | |
| echo "Existing channel_st3.json hash: $EXISTING_HASH" | |
| else | |
| echo "No existing channel_st3.json found or download failed" | |
| echo "existing_hash=none" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Determine if update is needed | |
| id: check | |
| run: | | |
| # If first run or hashes differ, update is needed | |
| if [ "${{ steps.download.outputs.existing_hash }}" != "${{ steps.generate.outputs.new_hash }}" ]; then | |
| echo "update_needed=true" >> $GITHUB_OUTPUT | |
| echo "Update needed: Content has changed or first run" | |
| else | |
| echo "update_needed=false" >> $GITHUB_OUTPUT | |
| echo "No update needed: Content is unchanged" | |
| fi | |
| - name: Update release and upload asset if needed | |
| if: steps.check.outputs.update_needed == 'true' | |
| run: | | |
| # Create or update the release | |
| gh release view ${{ env.RELEASE_TAG }} || \ | |
| gh release create ${{ env.RELEASE_TAG }} \ | |
| --title "The ST3 Channel" \ | |
| --notes "channel_st3.json" \ | |
| --latest=false | |
| echo "Uploading channel_st3.json file..." | |
| gh release upload ${{ env.RELEASE_TAG }} channel_st3.json --clobber | |
| # Format release notes | |
| HASH=${{ steps.generate.outputs.new_hash }} | |
| DATE=$(date -u +"%B %d, %Y, %H:%M UTC") | |
| REPO_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| # Build new notes | |
| { | |
| echo "channel_st3.json" | |
| echo "sha256: $HASH" | |
| echo "date: $DATE" | |
| echo "([logs]($REPO_URL))" | |
| echo "" | |
| echo "---" | |
| cat channel.log | |
| } > notes.txt | |
| echo "Updating release notes..." | |
| gh release edit ${{ env.RELEASE_TAG }} --notes-file notes.txt | |
| - name: Output result | |
| run: | | |
| if [ "${{ steps.check.outputs.update_needed }}" = "true" ]; then | |
| echo "✅ Channel updated with new content" | |
| else | |
| echo "ℹ️ No changes detected in channel_st3.json, skipped update" | |
| fi | |
| - name: Show diff (if update and not first run) | |
| if: steps.check.outputs.update_needed == 'true' && steps.download.outputs.existing_hash != 'none' | |
| run: | | |
| echo "📄 Showing diff between existing-channel_st3.json and channel_st3.json" | |
| python -m json.tool existing-channel_st3.json > old.json | |
| python -m json.tool channel_st3.json > new.json | |
| diff -u old.json new.json || true | |
| enrich_logs: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'workflow_run' || github.event_name == 'workflow_dispatch' | |
| continue-on-error: true | |
| env: | |
| RELEASE_TAG: crawler-status | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| WORKFLOW_ID: ${{ github.event.workflow_run.workflow_id }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: Install uv | |
| run: pip install uv | |
| - name: Ensure wrk directory exists | |
| run: mkdir -p ./wrk | |
| - name: Restore wrk cache | |
| id: restore-wrk-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ./wrk | |
| ./.pypi-cache | |
| key: wrk-cache-${{ github.run_id }} | |
| fail-on-cache-miss: false | |
| restore-keys: | | |
| wrk-cache- | |
| - name: Fetch workflow metadata for logs enrichment | |
| run: | | |
| uv run -m scripts.fetch_logs_metadata \ | |
| --runs-output "$RUNNER_TEMP/workflow_runs.json" \ | |
| --artifacts-output "$RUNNER_TEMP/workflow_artifacts.json" | |
| - name: Enrich logs.json | |
| run: | | |
| if [ ! -f ./wrk/logs.json ]; then | |
| echo "Missing cached ./wrk/logs.json; failing." >&2 | |
| exit 1 | |
| fi | |
| echo "Using cached ./wrk/logs.json" | |
| uv run -m scripts.enrich_logs \ | |
| --runs "$RUNNER_TEMP/workflow_runs.json" \ | |
| --artifacts "$RUNNER_TEMP/workflow_artifacts.json" \ | |
| ./wrk/logs.json | |
| retry_5() { | |
| local n=1 | |
| local max=5 | |
| local delay=2 | |
| while true; do | |
| "$@" && return 0 | |
| if [ "$n" -ge "$max" ]; then | |
| return 1 | |
| fi | |
| sleep $((delay + RANDOM % 3)) | |
| delay=$((delay * 2)) | |
| n=$((n + 1)) | |
| done | |
| } | |
| retry_5 gh release upload ${{ env.RELEASE_TAG }} ./wrk/logs.json --clobber | |
| website-after-build: | |
| runs-on: ubuntu-latest | |
| needs: [enrich_logs, build-st4-channel] | |
| if: needs.build-st4-channel.outputs.update_needed == 'true' | |
| steps: | |
| - name: Trigger website deployment | |
| env: | |
| SITE_DEPLOY_TOKEN: ${{ secrets.SITE_DEPLOY_TOKEN }} | |
| run: | | |
| curl --fail-with-body \ | |
| --request POST \ | |
| --url "https://api.github.com/repos/packagecontrol/website-stage/actions/workflows/deploy.yml/dispatches" \ | |
| --header "Accept: application/vnd.github+json" \ | |
| --header "Authorization: Bearer ${SITE_DEPLOY_TOKEN}" \ | |
| --header "X-GitHub-Api-Version: 2022-11-28" \ | |
| --data '{"ref":"main","inputs":{"source_ref":"gh-pages"}}' | |
| crawl-site-after-logs: | |
| runs-on: ubuntu-latest | |
| needs: enrich_logs | |
| permissions: | |
| actions: write | |
| contents: read | |
| steps: | |
| - name: Trigger crawl site deployment | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh workflow run deploy-crawl-site.yml \ | |
| --repo "${GITHUB_REPOSITORY}" \ | |
| --ref main \ | |
| --field source_ref=crawl-pages | |
| crawl-site-on-new-tags: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') | |
| permissions: | |
| actions: write | |
| contents: read | |
| steps: | |
| - name: Trigger crawl site deployment | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh workflow run deploy-crawl-site.yml \ | |
| --repo "${GITHUB_REPOSITORY}" \ | |
| --ref main \ | |
| --field source_ref=crawl-pages | |
| deploy-on-new-tags: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') | |
| steps: | |
| - name: Trigger website deployment | |
| env: | |
| SITE_DEPLOY_TOKEN: ${{ secrets.SITE_DEPLOY_TOKEN }} | |
| run: | | |
| curl --fail-with-body \ | |
| --request POST \ | |
| --url "https://api.github.com/repos/packagecontrol/website-stage/actions/workflows/deploy.yml/dispatches" \ | |
| --header "Accept: application/vnd.github+json" \ | |
| --header "Authorization: Bearer ${SITE_DEPLOY_TOKEN}" \ | |
| --header "X-GitHub-Api-Version: 2022-11-28" \ | |
| --data '{"ref":"main","inputs":{"source_ref":"gh-pages"}}' |