archive_logs_to_r2 #203
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: archive_logs_to_r2 | |
| on: | |
| workflow_run: | |
| workflows: ["*"] | |
| types: [completed] | |
| workflow_dispatch: | |
| permissions: | |
| actions: read | |
| jobs: | |
| archive-to-r2: | |
| runs-on: ubuntu-latest | |
| environment: prd | |
| # skip if the workflow run is this same workflow (self-archiving) | |
| if: ${{ github.event.workflow_run.name != github.workflow }} | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} | |
| R2_BUCKET: ${{ secrets.R2_BUCKET_NAME }} | |
| steps: | |
| - name: Download workflow run logs | |
| id: download | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const run = context.payload.workflow_run; | |
| const runId = run.id; | |
| const rawName = run.name || `workflow-${runId}`; | |
| const safeName = rawName.replace(/[^a-z0-9]/gi, '-').toLowerCase(); | |
| const ts = new Date(run.created_at).toISOString().replace(/[:.]/g, '-'); | |
| const base = 'archived-logs'; | |
| const dir = path.join(base, `${safeName}-${runId}-${ts}`); | |
| if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); | |
| const logs = await github.rest.actions.downloadWorkflowRunLogs({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: runId, | |
| }); | |
| // create filename in requested format: YYYYMMDD-<repo>-<workflow>-<runId> | |
| const repoName = context.repo.repo; | |
| const datePart = new Date(run.created_at).toISOString().split('T')[0].replace(/-/g, ''); // YYYYMMDD | |
| const filename = `${datePart}-${repoName}-${safeName}-${runId}.zip`; | |
| const fullPath = path.join(dir, filename); | |
| fs.writeFileSync(fullPath, Buffer.from(logs.data)); | |
| // export outputs and write to GITHUB_ENV so subsequent `run:` steps can use them | |
| core.setOutput('archive_path', fullPath); | |
| core.setOutput('archive_name', filename); | |
| core.setOutput('run_dir', dir); | |
| // write env entries directly to the environment file to avoid compile-time warnings | |
| if (process.env.GITHUB_ENV) { | |
| // ARCHIVE_PATH = path to zip; ARCHIVE_NAME = directory-style name without .zip | |
| const archiveNameDir = filename.replace(/\.zip$/i, ''); | |
| fs.appendFileSync(process.env.GITHUB_ENV, `ARCHIVE_PATH=${fullPath}\n`); | |
| fs.appendFileSync(process.env.GITHUB_ENV, `ARCHIVE_NAME=${archiveNameDir}\n`); | |
| // Export a sanitized workflow folder name so archives for the same workflow are grouped | |
| fs.appendFileSync(process.env.GITHUB_ENV, `WORKFLOW_SAFE=${safeName}\n`); | |
| } | |
| console.log(`Wrote logs to ${fullPath}`); | |
| - name: Ensure AWS CLI v2 is available | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # If aws is already present on the runner, use it. Otherwise download and install. | |
| if command -v aws >/dev/null 2>&1; then | |
| echo "aws exists: $(aws --version)" | |
| else | |
| echo "Installing AWS CLI..." | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
| unzip -q awscliv2.zip | |
| # use --update where appropriate; installer will succeed when there is no preexisting installation | |
| sudo ./aws/install --update || sudo ./aws/install | |
| aws --version | |
| fi | |
| - name: Upload logs | |
| shell: bash | |
| env: | |
| R2_ENDPOINT: ${{ env.R2_ENDPOINT }} | |
| R2_BUCKET: ${{ env.R2_BUCKET }} | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${ARCHIVE_PATH-}" ] || [ ! -f "${ARCHIVE_PATH}" ]; then | |
| echo "Archive not found: ${ARCHIVE_PATH}" | |
| exit 1 | |
| fi | |
| WORKDIR=$(mktemp -d) | |
| echo "Extracting ${ARCHIVE_PATH} to ${WORKDIR}..." | |
| unzip -q "${ARCHIVE_PATH}" -d "${WORKDIR}" | |
| # Destination prefix in the bucket: <workflow-folder>/<YYYYMMDD-<repo>-<workflow>-<run-id>/ | |
| # i.e. each workflow's runs are grouped under its own folder | |
| DEST_PREFIX="${WORKFLOW_SAFE}/${ARCHIVE_NAME}" | |
| echo "Syncing extracted logs to s3://${R2_BUCKET}/${DEST_PREFIX}/" | |
| # Use aws s3 sync to upload all files preserving folder structure | |
| aws s3 sync "${WORKDIR}/" "s3://${R2_BUCKET}/${DEST_PREFIX}/" --endpoint-url "${R2_ENDPOINT}" | |
| echo "Upload complete: s3://${R2_BUCKET}/${DEST_PREFIX}/" | |
| # export the destination for later steps to echo (folder prefix) | |
| echo "ARCHIVE_DEST=${DEST_PREFIX}" >> $GITHUB_ENV | |
| rm -rf "${WORKDIR}" | |
| # remove the zip from runner to save space | |
| rm -f "${ARCHIVE_PATH}" | |
| - name: Done | |
| run: | | |
| echo "Archive uploaded to R2: s3://${R2_BUCKET}/${ARCHIVE_DEST}" |