Merge pull request #432 from aojea/security-triage #352
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy SAM Hub | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - 'v*.*.*' | |
| workflow_dispatch: | |
| # Serialize deploys per-environment so a burst of pushes (e.g. several | |
| # dependabot PRs merging back-to-back) can't run concurrently against the | |
| # same GKE namespace and race on shared resources (e.g. the dns-sync-rollout | |
| # job name). Keeps only the latest queued run per environment; never cancels | |
| # one already in progress. | |
| concurrency: | |
| group: deploy-${{ github.ref_type == 'tag' && 'hub' || 'bananas' }} | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| env: | |
| REGISTRY: ghcr.io | |
| jobs: | |
| publish: | |
| name: Publish Images | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| contents: read | |
| timeout-minutes: 100 | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata for Control Plane | |
| id: meta-cp | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 | |
| with: | |
| images: ${{ env.REGISTRY }}/google/sam-control-plane | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=raw,value=${{ github.sha }} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} | |
| - name: Build and push Control Plane image | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 | |
| with: | |
| context: . | |
| file: Dockerfile.sam-control-plane | |
| platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| push: true | |
| tags: ${{ steps.meta-cp.outputs.tags }} | |
| labels: ${{ steps.meta-cp.outputs.labels }} | |
| - name: Extract metadata for Router | |
| id: meta-router | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 | |
| with: | |
| images: ${{ env.REGISTRY }}/google/sam-router | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=raw,value=${{ github.sha }} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} | |
| - name: Build and push Router image | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 | |
| with: | |
| context: . | |
| file: Dockerfile.sam-router | |
| platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| push: true | |
| tags: ${{ steps.meta-router.outputs.tags }} | |
| labels: ${{ steps.meta-router.outputs.labels }} | |
| - name: Extract metadata for Node | |
| id: meta-node | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 | |
| with: | |
| images: ${{ env.REGISTRY }}/google/sam-node | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=raw,value=${{ github.sha }} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} | |
| - name: Build and push Node image | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 | |
| with: | |
| context: . | |
| file: Dockerfile.sam-node | |
| platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| push: true | |
| tags: ${{ steps.meta-node.outputs.tags }} | |
| labels: ${{ steps.meta-node.outputs.labels }} | |
| - name: Extract metadata for nano-init | |
| id: meta-nano-init | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 | |
| with: | |
| images: ${{ env.REGISTRY }}/google/sam-nano-init | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=raw,value=${{ github.sha }} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} | |
| - name: Build and push nano-init image | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 | |
| with: | |
| context: . | |
| file: Dockerfile.nano-init | |
| platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| push: true | |
| tags: ${{ steps.meta-nano-init.outputs.tags }} | |
| labels: ${{ steps.meta-nano-init.outputs.labels }} | |
| - name: Extract metadata for sam-box | |
| id: meta-sambox | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 | |
| with: | |
| images: ${{ env.REGISTRY }}/google/sam-box | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=raw,value=${{ github.sha }} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} | |
| - name: Build and push sam-box image | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 | |
| with: | |
| context: . | |
| file: Dockerfile.sam-box | |
| platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| push: true | |
| tags: ${{ steps.meta-sambox.outputs.tags }} | |
| labels: ${{ steps.meta-sambox.outputs.labels }} | |
| - name: Extract metadata for sam-console | |
| id: meta-console | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 | |
| with: | |
| images: ${{ env.REGISTRY }}/google/sam-console | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=raw,value=${{ github.sha }} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} | |
| - name: Build and push sam-console image | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 | |
| with: | |
| context: . | |
| file: Dockerfile.sam-console | |
| platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| push: true | |
| tags: ${{ steps.meta-console.outputs.tags }} | |
| labels: ${{ steps.meta-console.outputs.labels }} | |
| deploy: | |
| name: Deploy to GKE | |
| runs-on: ubuntu-latest | |
| needs: publish | |
| # Dynamically select the GitHub Environment based on the trigger | |
| environment: ${{ github.ref_type == 'tag' && 'hub' || 'bananas' }} | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Determine Image Tag | |
| env: | |
| REF_TYPE: ${{ github.ref_type }} | |
| REF_NAME: ${{ github.ref_name }} | |
| COMMIT_SHA: ${{ github.sha }} | |
| run: | | |
| if [[ "${REF_TYPE}" == "tag" ]]; then | |
| echo "IMAGE_TAG=${REF_NAME}" >> $GITHUB_ENV | |
| else | |
| echo "IMAGE_TAG=${COMMIT_SHA}" >> $GITHUB_ENV | |
| fi | |
| - name: Google Auth | |
| uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2 | |
| with: | |
| workload_identity_provider: ${{ vars.WIF_PROVIDER_NAME }} | |
| service_account: ${{ vars.SERVICE_ACCOUNT_EMAIL }} | |
| - name: Set up GKE credentials | |
| uses: google-github-actions/get-gke-credentials@3da1e46a907576cefaa90c484278bb5b259dd395 # v3.0.0 | |
| with: | |
| cluster_name: ${{ vars.CLUSTER_NAME }} | |
| location: ${{ vars.CLUSTER_REGION }} | |
| - name: Prepare Namespaces | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| run: | | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| kubectl create namespace dex --dry-run=client -o yaml | kubectl apply -f - | |
| kubectl create namespace ${NAMESPACE} --dry-run=client -o yaml | kubectl apply -f - | |
| kubectl create namespace ${CANARY_NAMESPACE} --dry-run=client -o yaml | kubectl apply -f - | |
| - name: Provision Dex Secrets | |
| env: | |
| GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} | |
| GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} | |
| GITHUB_CLIENT_ID: ${{ secrets.GHUB_CLIENT_ID }} | |
| GITHUB_CLIENT_SECRET: ${{ secrets.GHUB_CLIENT_SECRET }} | |
| CLI_OAUTH_SECRET: ${{ secrets.CLI_OAUTH_SECRET }} | |
| run: | | |
| # Declaratively apply secrets without writing them to disk | |
| kubectl create secret generic dex-secrets \ | |
| --namespace=dex \ | |
| --from-literal=google-client-id="${GOOGLE_CLIENT_ID}" \ | |
| --from-literal=google-client-secret="${GOOGLE_CLIENT_SECRET}" \ | |
| --from-literal=github-client-id="${GITHUB_CLIENT_ID}" \ | |
| --from-literal=github-client-secret="${GITHUB_CLIENT_SECRET}" \ | |
| --from-literal=cli-oauth-secret="${CLI_OAUTH_SECRET}" \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| - name: Deploy Dex Resources | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| # Apply the Dex manifests | |
| kubectl apply -f .github/k8s/dex-config.yaml | |
| kubectl apply -f .github/k8s/dex-deployment.yaml | |
| # Force a restart to pick up any changes to the secrets, and wait for readiness | |
| kubectl rollout restart deployment/dex -n dex | |
| kubectl rollout status deployment/dex -n dex --timeout=120s || { | |
| echo "Dex Deployment failed!" | |
| print_rollout_diagnostics "dex" "dex" "app=dex" | |
| echo "Rolling back Dex deployment..." | |
| kubectl rollout undo deployment/dex -n dex | |
| exit 1 | |
| } | |
| - name: Provision Hub Secrets | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| run: | | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| # Idempotently provision the sam-hub-secret if it doesn't already exist | |
| if ! kubectl get secret sam-hub-secret -n ${NAMESPACE} >/dev/null 2>&1; then | |
| echo "Generating a new random 32-byte hex key for SAM_HUB_KEY..." | |
| HUB_KEY=$(openssl rand -hex 32) | |
| kubectl create secret generic sam-hub-secret \ | |
| --namespace=${NAMESPACE} \ | |
| --from-literal=sam-hub-key="${HUB_KEY}" | |
| else | |
| echo "sam-hub-secret already exists in namespace ${NAMESPACE}. Skipping generation." | |
| fi | |
| - name: Provision Control Plane Secrets | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| run: | | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| if ! kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} >/dev/null 2>&1; then | |
| echo "Generating new random secrets for admin-token and the DB..." | |
| ADMIN_TOKEN=$(openssl rand -hex 32) | |
| DB_PASSWORD=$(openssl rand -hex 32) | |
| DB_DSN="postgres://sam:${DB_PASSWORD}@sam-db-${ENV_NAME}:5432/sam_mesh?sslmode=disable" | |
| kubectl create secret generic sam-control-plane-secret-${ENV_NAME} \ | |
| --namespace=${NAMESPACE} \ | |
| --from-literal=admin-token="${ADMIN_TOKEN}" \ | |
| --from-literal=db-password="${DB_PASSWORD}" \ | |
| --from-literal=db-dsn="${DB_DSN}" | |
| else | |
| echo "sam-control-plane-secret-${ENV_NAME} already exists in namespace ${NAMESPACE}. Skipping generation." | |
| fi | |
| - name: Deploy Control Plane and Routers | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| VAR_GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} | |
| VAR_CLUSTER_REGION: ${{ vars.CLUSTER_REGION }} | |
| VAR_CLUSTER_NAME: ${{ vars.CLUSTER_NAME }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local resource="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for ${resource} in namespace ${namespace}..." | |
| kubectl describe ${resource} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| sudo apt-get install -y gettext-base | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| export GCP_PROJECT_ID="${VAR_GCP_PROJECT_ID}" | |
| export CLUSTER_REGION="${VAR_CLUSTER_REGION}" | |
| export CLUSTER_NAME="${VAR_CLUSTER_NAME}" | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-control-plane-template.yaml | kubectl apply -f - | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-router-template.yaml | kubectl apply -f - | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-console-template.yaml | kubectl apply -f - | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID}' < .github/k8s/dns-sync-cronjob-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/sam-control-plane-${ENV_NAME} -n ${NAMESPACE} --timeout=120s || { | |
| echo "Control Plane Deployment failed!" | |
| print_rollout_diagnostics "${NAMESPACE}" "deployment/sam-control-plane-${ENV_NAME}" "app=sam-control-plane-${ENV_NAME}" | |
| exit 1 | |
| } | |
| echo "Seeding Control Plane Policies..." | |
| ADMIN_TOKEN=$(kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} -o jsonpath='{.data.admin-token}' | base64 -d) | |
| echo "::add-mask::${ADMIN_TOKEN}" | |
| kubectl port-forward deployment/sam-control-plane-${ENV_NAME} -n ${NAMESPACE} 8080:8080 & | |
| PF_PID=$! | |
| for i in {1..15}; do | |
| if nc -z localhost 8080; then | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| curl -fsS -X POST \ | |
| -H "Content-Type: application/json" \ | |
| -H "Authorization: Bearer ${ADMIN_TOKEN}" \ | |
| -d '{ | |
| "roles": [ | |
| {"name": "sam-canary", "allowed_services": ["*"], "allowed_targets": ["*"]}, | |
| {"name": "sam:role:sambox", "allowed_services": ["*"], "allowed_targets": ["*"]}, | |
| {"name": "sam:role:router", "allowed_services": ["*"], "allowed_targets": ["*"]}, | |
| {"name": "sam:role:node"}, | |
| {"name": "public-mesh", "allowed_services": ["*"], "allowed_targets": ["*"]} | |
| ], | |
| "bindings": [ | |
| {"role": "sam-canary", "members": ["user:system:serviceaccount:sam-canary-'"${ENV_NAME}"':sam-node-sa", "user:system:serviceaccount:sam-canary-'"${ENV_NAME}"':sam-box-sa"]}, | |
| {"role": "sam:role:router", "members": ["group:routers", "user:system:serviceaccount:'"${NAMESPACE}"':sam-router-sa"]}, | |
| {"role": "sam:role:node", "members": ["sam:system:authenticated"]}, | |
| {"role": "public-mesh", "members": ["sam:system:authenticated"]} | |
| ] | |
| }' \ | |
| http://localhost:8080/policies || { | |
| echo "Policy seeding failed - a rejected policy leaves enrollment broken (nodes get 403 at /register)." | |
| kill $PF_PID | |
| exit 1 | |
| } | |
| kill $PF_PID | |
| kubectl rollout status deployment/sam-console-${ENV_NAME} -n ${NAMESPACE} --timeout=120s || { | |
| echo "Console Deployment failed!" | |
| print_rollout_diagnostics "${NAMESPACE}" "deployment/sam-console-${ENV_NAME}" "app=sam-console-${ENV_NAME}" | |
| exit 1 | |
| } | |
| kubectl rollout status statefulset/sam-router-${ENV_NAME} -n ${NAMESPACE} --timeout=120s || { | |
| echo "Router Deployment failed!" | |
| print_rollout_diagnostics "${NAMESPACE}" "statefulset/sam-router-${ENV_NAME}" "app=sam-router-${ENV_NAME}" | |
| echo "Rolling back Router deployment..." | |
| kubectl rollout undo statefulset/sam-router-${ENV_NAME} -n ${NAMESPACE} | |
| kubectl rollout status statefulset/sam-router-${ENV_NAME} -n ${NAMESPACE} | |
| exit 1 | |
| } | |
| echo "Triggering immediate DNS sync after rollout..." | |
| kubectl delete job dns-sync-rollout-${ENV_NAME} -n ${NAMESPACE} --ignore-not-found || true | |
| kubectl create job --from=cronjob/dns-sync-cronjob dns-sync-rollout-${ENV_NAME} -n ${NAMESPACE} | |
| echo "Waiting for DNS sync job to complete..." | |
| kubectl wait --for=condition=complete job/dns-sync-rollout-${ENV_NAME} -n ${NAMESPACE} --timeout=180s || { | |
| echo "DNS sync job failed or timed out!" | |
| kubectl describe job/dns-sync-rollout-${ENV_NAME} -n ${NAMESPACE} | |
| kubectl logs -n ${NAMESPACE} -l job-name=dns-sync-rollout-${ENV_NAME} --tail=-1 || true | |
| exit 1 | |
| } | |
| - name: Deploy GCE VM Router | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| VAR_GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} | |
| VAR_CLUSTER_REGION: ${{ vars.CLUSTER_REGION }} | |
| run: | | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| export GCP_PROJECT_ID="${VAR_GCP_PROJECT_ID}" | |
| export CLUSTER_REGION="${VAR_CLUSTER_REGION}" | |
| export ZONE="${CLUSTER_REGION}-a" | |
| echo "Retrieving Admin Token to generate Bootstrap Token..." | |
| ADMIN_TOKEN=$(kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} -o jsonpath='{.data.admin-token}' | base64 -d) | |
| echo "::add-mask::${ADMIN_TOKEN}" | |
| # Mint the token from the runner over a port-forward: a helper pod | |
| # would carry the admin token in its spec, where kubectl describe | |
| # and anyone with pod read access can see it. | |
| echo "Generating Bootstrap Token via port-forward to the Control Plane..." | |
| kubectl port-forward deployment/sam-control-plane-${ENV_NAME} -n ${NAMESPACE} 8080:8080 & | |
| PF_PID=$! | |
| for i in {1..15}; do | |
| if nc -z localhost 8080; then | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| TOKEN_JSON=$(curl -fsS -X POST \ | |
| -H "Content-Type: application/json" \ | |
| -H "Authorization: Bearer ${ADMIN_TOKEN}" \ | |
| -d '{"role": "sam:role:router", "max_usages": 1}' \ | |
| http://localhost:8080/admin/bootstrap-tokens) || { | |
| echo "ERROR: bootstrap token request failed" | |
| kill $PF_PID | |
| exit 1 | |
| } | |
| kill $PF_PID | |
| BOOTSTRAP_TOKEN=$(echo "${TOKEN_JSON}" | jq -r .token) | |
| if [ -z "${BOOTSTRAP_TOKEN}" ] || [ "${BOOTSTRAP_TOKEN}" = "null" ]; then | |
| echo "ERROR: Generated bootstrap token is empty or invalid!" | |
| echo "Response (token redacted): $(echo "${TOKEN_JSON}" | jq -c 'del(.token)' 2>/dev/null || echo '<not JSON>')" | |
| exit 1 | |
| fi | |
| echo "::add-mask::${BOOTSTRAP_TOKEN}" | |
| echo "Successfully generated bootstrap token." | |
| echo "Provisioning GCE VM Router instance..." | |
| # Delete existing VM if it exists to ensure fresh deployment with the new token and image | |
| gcloud compute instances delete sam-router-${ENV_NAME}-vm --zone=${ZONE} --project=${GCP_PROJECT_ID} --quiet || true | |
| # Create a new COS VM instance running the sam-router | |
| gcloud compute instances create sam-router-${ENV_NAME}-vm \ | |
| --project=${GCP_PROJECT_ID} \ | |
| --zone=${ZONE} \ | |
| --machine-type=e2-micro \ | |
| --image-family=cos-stable \ | |
| --image-project=cos-cloud \ | |
| --tags=sam-router \ | |
| --metadata=startup-script='#!/bin/bash | |
| set -euo pipefail | |
| EXTERNAL_IP=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip) | |
| docker run -d \ | |
| --name sam-router \ | |
| --network host \ | |
| --restart always \ | |
| ghcr.io/google/sam-router:'"${IMAGE_TAG}"' \ | |
| --control-plane=https://'"${ENV_NAME}"'.sam-mesh.dev \ | |
| --bootstrap-token='"${BOOTSTRAP_TOKEN}"' \ | |
| --listen=/ip4/0.0.0.0/tcp/4501 \ | |
| --listen=/ip4/0.0.0.0/udp/4501/quic-v1 \ | |
| --external-addr=/ip4/${EXTERNAL_IP}/tcp/4501 \ | |
| --external-addr=/ip4/${EXTERNAL_IP}/udp/4501/quic-v1' | |
| - name: Deploy COP Canary | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| export BANANA_BOT_SCRIPT=$(cat site/content/docs/snippets/banana_bot_playground.py | sed 's/^/ /') | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${IMAGE_TAG} ${BANANA_BOT_SCRIPT}' < .github/k8s/sam-node-cop-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/cop-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || { | |
| echo "Cop Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "cop-canary-${ENV_NAME}" "app=cop-canary-${ENV_NAME}" | |
| exit 1 | |
| } | |
| - name: Deploy SAM Box Canary | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${IMAGE_TAG}' < .github/k8s/sam-box-canary-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/box-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || { | |
| echo "sam-box Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "box-canary-${ENV_NAME}" "app=box-canary-${ENV_NAME}" | |
| exit 1 | |
| } | |
| - name: Deploy SAM Node Canary | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${IMAGE_TAG}' < .github/k8s/sam-node-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/sam-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || { | |
| echo "Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "sam-canary-${ENV_NAME}" "app=sam-canary-${ENV_NAME}" | |
| echo "Rolling back Canary deployment..." | |
| kubectl rollout undo deployment/sam-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} | |
| kubectl rollout status deployment/sam-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true | |
| exit 1 | |
| } | |
| - name: Deploy OpenClaw Canary | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| VAR_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| export GEMINI_API_KEY="${VAR_GEMINI_API_KEY}" | |
| GATEWAY_TOKEN=$(kubectl get secret openclaw-secret-${ENV_NAME} -n ${CANARY_NAMESPACE} -o jsonpath="{.data.gateway-token}" 2>/dev/null | base64 -d || true) | |
| # base64 -d on empty input exits 0, so a missing secret used to yield | |
| # an empty token here; test the value, not the pipeline status. | |
| if [ -z "${GATEWAY_TOKEN}" ]; then | |
| GATEWAY_TOKEN=$(openssl rand -hex 16) | |
| fi | |
| echo "::add-mask::${GATEWAY_TOKEN}" | |
| kubectl create secret generic openclaw-secret-${ENV_NAME} \ | |
| --namespace=${CANARY_NAMESPACE} \ | |
| --from-literal=gateway-token="${GATEWAY_TOKEN}" \ | |
| --from-literal=gemini-api-key="${GEMINI_API_KEY}" \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| # Allow-list the placeholders: ${OPENCLAW_GATEWAY_TOKEN} in the | |
| # template is a container-runtime shell variable, not ours to expand. | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${IMAGE_TAG}' < .github/k8s/sam-node-openclaw-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/openclaw-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=300s || { | |
| echo "OpenClaw Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "openclaw-canary-${ENV_NAME}" "app=openclaw-canary-${ENV_NAME}" | |
| echo "Rolling back OpenClaw Canary deployment..." | |
| kubectl rollout undo deployment/openclaw-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} | |
| kubectl rollout status deployment/openclaw-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true | |
| exit 1 | |
| } | |
| - name: Deploy vLLM Canary | |
| if: vars.ENV_NAME != 'hub' | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| VAR_HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| export HF_TOKEN="${VAR_HF_TOKEN}" | |
| kubectl create secret generic vllm-secret-${ENV_NAME} \ | |
| --namespace=${CANARY_NAMESPACE} \ | |
| --from-literal=hf-token="${HF_TOKEN}" \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${IMAGE_TAG}' < .github/k8s/sam-node-vllm-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/vllm-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=300s || { | |
| echo "vLLM Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "vllm-canary-${ENV_NAME}" "app=vllm-canary-${ENV_NAME}" | |
| echo "Rolling back vLLM Canary deployment..." | |
| kubectl rollout undo deployment/vllm-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} | |
| kubectl rollout status deployment/vllm-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true | |
| exit 1 | |
| } | |
| - name: Deploy Everything Canary | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${IMAGE_TAG}' < .github/k8s/sam-node-everything-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/everything-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || { | |
| echo "Everything Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "everything-canary-${ENV_NAME}" "app=everything-canary-${ENV_NAME}" | |
| echo "Rolling back Everything Canary deployment..." | |
| kubectl rollout undo deployment/everything-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} | |
| kubectl rollout status deployment/everything-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true | |
| exit 1 | |
| } | |
| - name: Deploy OpenRouter Node to Testnet | |
| env: | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| # The key lives in a Secret the pod references, never in the | |
| # Deployment spec. | |
| kubectl create secret generic openrouter-secret-${ENV_NAME} \ | |
| --namespace=${CANARY_NAMESPACE} \ | |
| --from-literal=api-key="${OPENROUTER_API_KEY}" \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${IMAGE_TAG}' < .github/k8s/sam-node-openrouter-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/openrouter-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || { | |
| echo "OpenRouter Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "openrouter-canary-${ENV_NAME}" "app=openrouter-canary-${ENV_NAME}" | |
| echo "Rolling back OpenRouter Canary deployment..." | |
| kubectl rollout undo deployment/openrouter-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} | |
| kubectl rollout status deployment/openrouter-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true | |
| exit 1 | |
| } |