Merge pull request #365 from aojea/sam-one-join-bind-addr #300
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy SAM Hub | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - 'v*.*.*' | |
| workflow_dispatch: | |
| # Serialize deploys per-environment so a burst of pushes (e.g. several | |
| # dependabot PRs merging back-to-back) can't run concurrently against the | |
| # same GKE namespace and race on shared resources (e.g. the dns-sync-rollout | |
| # job name). Keeps only the latest queued run per environment; never cancels | |
| # one already in progress. | |
| concurrency: | |
| group: deploy-${{ github.ref_type == 'tag' && 'hub' || 'bananas' }} | |
| cancel-in-progress: false | |
| env: | |
| REGISTRY: ghcr.io | |
| jobs: | |
| publish: | |
| name: Publish Images | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| contents: read | |
| timeout-minutes: 100 | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata for Control Plane | |
| id: meta-cp | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 | |
| with: | |
| images: ${{ env.REGISTRY }}/google/sam-control-plane | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=raw,value=${{ github.sha }} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} | |
| - name: Build and push Control Plane image | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 | |
| with: | |
| context: . | |
| file: Dockerfile.sam-control-plane | |
| platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| push: true | |
| tags: ${{ steps.meta-cp.outputs.tags }} | |
| labels: ${{ steps.meta-cp.outputs.labels }} | |
| - name: Extract metadata for Router | |
| id: meta-router | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 | |
| with: | |
| images: ${{ env.REGISTRY }}/google/sam-router | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=raw,value=${{ github.sha }} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} | |
| - name: Build and push Router image | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 | |
| with: | |
| context: . | |
| file: Dockerfile.sam-router | |
| platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| push: true | |
| tags: ${{ steps.meta-router.outputs.tags }} | |
| labels: ${{ steps.meta-router.outputs.labels }} | |
| - name: Extract metadata for Node | |
| id: meta-node | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 | |
| with: | |
| images: ${{ env.REGISTRY }}/google/sam-node | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=raw,value=${{ github.sha }} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} | |
| - name: Build and push Node image | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 | |
| with: | |
| context: . | |
| file: Dockerfile.sam-node | |
| platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| push: true | |
| tags: ${{ steps.meta-node.outputs.tags }} | |
| labels: ${{ steps.meta-node.outputs.labels }} | |
| - name: Extract metadata for nano-init | |
| id: meta-nano-init | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 | |
| with: | |
| images: ${{ env.REGISTRY }}/google/sam-nano-init | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=raw,value=${{ github.sha }} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} | |
| - name: Build and push nano-init image | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 | |
| with: | |
| context: . | |
| file: Dockerfile.nano-init | |
| platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| push: true | |
| tags: ${{ steps.meta-nano-init.outputs.tags }} | |
| labels: ${{ steps.meta-nano-init.outputs.labels }} | |
| - name: Extract metadata for sam-box | |
| id: meta-sambox | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 | |
| with: | |
| images: ${{ env.REGISTRY }}/google/sam-box | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=raw,value=${{ github.sha }} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} | |
| - name: Build and push sam-box image | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 | |
| with: | |
| context: . | |
| file: Dockerfile.sam-box | |
| platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| push: true | |
| tags: ${{ steps.meta-sambox.outputs.tags }} | |
| labels: ${{ steps.meta-sambox.outputs.labels }} | |
| - name: Extract metadata for sam-console | |
| id: meta-console | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 | |
| with: | |
| images: ${{ env.REGISTRY }}/google/sam-console | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=raw,value=${{ github.sha }} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} | |
| - name: Build and push sam-console image | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 | |
| with: | |
| context: . | |
| file: Dockerfile.sam-console | |
| platforms: ${{ github.ref_type == 'tag' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| push: true | |
| tags: ${{ steps.meta-console.outputs.tags }} | |
| labels: ${{ steps.meta-console.outputs.labels }} | |
| deploy: | |
| name: Deploy to GKE | |
| runs-on: ubuntu-latest | |
| needs: publish | |
| # Dynamically select the GitHub Environment based on the trigger | |
| environment: ${{ github.ref_type == 'tag' && 'hub' || 'bananas' }} | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Determine Image Tag | |
| env: | |
| REF_TYPE: ${{ github.ref_type }} | |
| REF_NAME: ${{ github.ref_name }} | |
| COMMIT_SHA: ${{ github.sha }} | |
| run: | | |
| if [[ "${REF_TYPE}" == "tag" ]]; then | |
| echo "IMAGE_TAG=${REF_NAME}" >> $GITHUB_ENV | |
| else | |
| echo "IMAGE_TAG=${COMMIT_SHA}" >> $GITHUB_ENV | |
| fi | |
| - name: Google Auth | |
| uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2 | |
| with: | |
| workload_identity_provider: ${{ vars.WIF_PROVIDER_NAME }} | |
| service_account: ${{ vars.SERVICE_ACCOUNT_EMAIL }} | |
| - name: Set up GKE credentials | |
| uses: google-github-actions/get-gke-credentials@3da1e46a907576cefaa90c484278bb5b259dd395 # v3.0.0 | |
| with: | |
| cluster_name: ${{ vars.CLUSTER_NAME }} | |
| location: ${{ vars.CLUSTER_REGION }} | |
| - name: Prepare Namespaces | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| run: | | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| kubectl create namespace dex --dry-run=client -o yaml | kubectl apply -f - | |
| kubectl create namespace ${NAMESPACE} --dry-run=client -o yaml | kubectl apply -f - | |
| kubectl create namespace ${CANARY_NAMESPACE} --dry-run=client -o yaml | kubectl apply -f - | |
| - name: Provision Dex Secrets | |
| env: | |
| GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} | |
| GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} | |
| GITHUB_CLIENT_ID: ${{ secrets.GHUB_CLIENT_ID }} | |
| GITHUB_CLIENT_SECRET: ${{ secrets.GHUB_CLIENT_SECRET }} | |
| CLI_OAUTH_SECRET: ${{ secrets.CLI_OAUTH_SECRET }} | |
| run: | | |
| # Declaratively apply secrets without writing them to disk | |
| kubectl create secret generic dex-secrets \ | |
| --namespace=dex \ | |
| --from-literal=google-client-id="${GOOGLE_CLIENT_ID}" \ | |
| --from-literal=google-client-secret="${GOOGLE_CLIENT_SECRET}" \ | |
| --from-literal=github-client-id="${GITHUB_CLIENT_ID}" \ | |
| --from-literal=github-client-secret="${GITHUB_CLIENT_SECRET}" \ | |
| --from-literal=cli-oauth-secret="${CLI_OAUTH_SECRET}" \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| - name: Deploy Dex Resources | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| # Apply the Dex manifests | |
| kubectl apply -f .github/k8s/dex-config.yaml | |
| kubectl apply -f .github/k8s/dex-deployment.yaml | |
| # Force a restart to pick up any changes to the secrets, and wait for readiness | |
| kubectl rollout restart deployment/dex -n dex | |
| kubectl rollout status deployment/dex -n dex --timeout=120s || { | |
| echo "Dex Deployment failed!" | |
| print_rollout_diagnostics "dex" "dex" "app=dex" | |
| echo "Rolling back Dex deployment..." | |
| kubectl rollout undo deployment/dex -n dex | |
| exit 1 | |
| } | |
| - name: Provision Hub Secrets | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| run: | | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| # Idempotently provision the sam-hub-secret if it doesn't already exist | |
| if ! kubectl get secret sam-hub-secret -n ${NAMESPACE} >/dev/null 2>&1; then | |
| echo "Generating a new random 32-byte hex key for SAM_HUB_KEY..." | |
| HUB_KEY=$(openssl rand -hex 32) | |
| kubectl create secret generic sam-hub-secret \ | |
| --namespace=${NAMESPACE} \ | |
| --from-literal=sam-hub-key="${HUB_KEY}" | |
| else | |
| echo "sam-hub-secret already exists in namespace ${NAMESPACE}. Skipping generation." | |
| fi | |
| - name: Provision Control Plane Secrets | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| run: | | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| if ! kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} >/dev/null 2>&1; then | |
| echo "Generating new random secrets for admin-token and the DB..." | |
| ADMIN_TOKEN=$(openssl rand -hex 32) | |
| DB_PASSWORD=$(openssl rand -hex 32) | |
| DB_DSN="postgres://sam:${DB_PASSWORD}@sam-db-${ENV_NAME}:5432/sam_mesh?sslmode=disable" | |
| kubectl create secret generic sam-control-plane-secret-${ENV_NAME} \ | |
| --namespace=${NAMESPACE} \ | |
| --from-literal=admin-token="${ADMIN_TOKEN}" \ | |
| --from-literal=db-password="${DB_PASSWORD}" \ | |
| --from-literal=db-dsn="${DB_DSN}" | |
| else | |
| echo "sam-control-plane-secret-${ENV_NAME} already exists in namespace ${NAMESPACE}. Skipping generation." | |
| fi | |
| - name: Deploy Control Plane and Routers | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| VAR_GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} | |
| VAR_CLUSTER_REGION: ${{ vars.CLUSTER_REGION }} | |
| VAR_CLUSTER_NAME: ${{ vars.CLUSTER_NAME }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local resource="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for ${resource} in namespace ${namespace}..." | |
| kubectl describe ${resource} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| sudo apt-get install -y gettext-base | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| export GCP_PROJECT_ID="${VAR_GCP_PROJECT_ID}" | |
| export CLUSTER_REGION="${VAR_CLUSTER_REGION}" | |
| export CLUSTER_NAME="${VAR_CLUSTER_NAME}" | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-control-plane-template.yaml | kubectl apply -f - | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-router-template.yaml | kubectl apply -f - | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/sam-console-template.yaml | kubectl apply -f - | |
| envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID}' < .github/k8s/dns-sync-cronjob-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/sam-control-plane-${ENV_NAME} -n ${NAMESPACE} --timeout=120s || { | |
| echo "Control Plane Deployment failed!" | |
| print_rollout_diagnostics "${NAMESPACE}" "deployment/sam-control-plane-${ENV_NAME}" "app=sam-control-plane-${ENV_NAME}" | |
| exit 1 | |
| } | |
| echo "Seeding Control Plane Policies..." | |
| ADMIN_TOKEN=$(kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} -o jsonpath='{.data.admin-token}' | base64 -d) | |
| kubectl port-forward deployment/sam-control-plane-${ENV_NAME} -n ${NAMESPACE} 8080:8080 & | |
| PF_PID=$! | |
| for i in {1..15}; do | |
| if nc -z localhost 8080; then | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| curl -fsS -X POST \ | |
| -H "Content-Type: application/json" \ | |
| -H "Authorization: Bearer ${ADMIN_TOKEN}" \ | |
| -d '{ | |
| "roles": [ | |
| {"name": "sam-canary", "allowed_services": ["*"], "allowed_targets": ["*"]}, | |
| {"name": "sam:role:sambox", "allowed_services": ["*"], "allowed_targets": ["*"]}, | |
| {"name": "sam:role:router", "allowed_services": ["*"], "allowed_targets": ["*"]}, | |
| {"name": "sam:role:node"}, | |
| {"name": "public-mesh", "allowed_services": ["*"], "allowed_targets": ["*"]} | |
| ], | |
| "bindings": [ | |
| {"role": "sam-canary", "members": ["user:system:serviceaccount:sam-canary-'"${ENV_NAME}"':sam-node-sa", "user:system:serviceaccount:sam-canary-'"${ENV_NAME}"':sam-box-sa"]}, | |
| {"role": "sam:role:router", "members": ["group:routers", "user:system:serviceaccount:'"${NAMESPACE}"':sam-router-sa"]}, | |
| {"role": "sam:role:node", "members": ["sam:system:authenticated"]}, | |
| {"role": "public-mesh", "members": ["sam:system:authenticated"]} | |
| ] | |
| }' \ | |
| http://localhost:8080/policies || { | |
| echo "Policy seeding failed - a rejected policy leaves enrollment broken (nodes get 403 at /register)." | |
| kill $PF_PID | |
| exit 1 | |
| } | |
| kill $PF_PID | |
| kubectl rollout status deployment/sam-console-${ENV_NAME} -n ${NAMESPACE} --timeout=120s || { | |
| echo "Console Deployment failed!" | |
| print_rollout_diagnostics "${NAMESPACE}" "deployment/sam-console-${ENV_NAME}" "app=sam-console-${ENV_NAME}" | |
| exit 1 | |
| } | |
| kubectl rollout status statefulset/sam-router-${ENV_NAME} -n ${NAMESPACE} --timeout=120s || { | |
| echo "Router Deployment failed!" | |
| print_rollout_diagnostics "${NAMESPACE}" "statefulset/sam-router-${ENV_NAME}" "app=sam-router-${ENV_NAME}" | |
| echo "Rolling back Router deployment..." | |
| kubectl rollout undo statefulset/sam-router-${ENV_NAME} -n ${NAMESPACE} | |
| kubectl rollout status statefulset/sam-router-${ENV_NAME} -n ${NAMESPACE} | |
| exit 1 | |
| } | |
| echo "Triggering immediate DNS sync after rollout..." | |
| kubectl delete job dns-sync-rollout-${ENV_NAME} -n ${NAMESPACE} --ignore-not-found || true | |
| kubectl create job --from=cronjob/dns-sync-cronjob dns-sync-rollout-${ENV_NAME} -n ${NAMESPACE} | |
| echo "Waiting for DNS sync job to complete..." | |
| kubectl wait --for=condition=complete job/dns-sync-rollout-${ENV_NAME} -n ${NAMESPACE} --timeout=180s || { | |
| echo "DNS sync job failed or timed out!" | |
| kubectl describe job/dns-sync-rollout-${ENV_NAME} -n ${NAMESPACE} | |
| kubectl logs -n ${NAMESPACE} -l job-name=dns-sync-rollout-${ENV_NAME} --tail=-1 || true | |
| exit 1 | |
| } | |
| - name: Deploy GCE VM Router | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| VAR_GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} | |
| VAR_CLUSTER_REGION: ${{ vars.CLUSTER_REGION }} | |
| run: | | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| export GCP_PROJECT_ID="${VAR_GCP_PROJECT_ID}" | |
| export CLUSTER_REGION="${VAR_CLUSTER_REGION}" | |
| export ZONE="${CLUSTER_REGION}-a" | |
| echo "Retrieving Admin Token to generate Bootstrap Token..." | |
| ADMIN_TOKEN=$(kubectl get secret sam-control-plane-secret-${ENV_NAME} -n ${NAMESPACE} -o jsonpath='{.data.admin-token}' | base64 -d) | |
| echo "Generating Bootstrap Token via internal GCP/GKE Control Plane service..." | |
| # Clean up any leftover token generator pod | |
| kubectl delete pod curl-token-gen-vm -n ${NAMESPACE} --ignore-not-found || true | |
| # Run temporary curl pod in control plane namespace | |
| kubectl run curl-token-gen-vm \ | |
| -n ${NAMESPACE} \ | |
| --image=curlimages/curl:8.6.0 \ | |
| --restart=Never \ | |
| --overrides='{"spec": {"activeDeadlineSeconds": 30}}' \ | |
| -- \ | |
| curl -s -X POST \ | |
| -H "Content-Type: application/json" \ | |
| -H "Authorization: Bearer ${ADMIN_TOKEN}" \ | |
| -d '{"role": "sam:role:router", "max_usages": 1}' \ | |
| http://sam-control-plane-${ENV_NAME}:8080/admin/bootstrap-tokens | |
| if ! kubectl wait -n ${NAMESPACE} --for=jsonpath='{.status.phase}'=Succeeded pod/curl-token-gen-vm --timeout=15s; then | |
| echo "ERROR: Token generation pod failed!" | |
| kubectl describe pod curl-token-gen-vm -n ${NAMESPACE} || true | |
| kubectl logs pod/curl-token-gen-vm -n ${NAMESPACE} || true | |
| kubectl delete pod curl-token-gen-vm -n ${NAMESPACE} --ignore-not-found || true | |
| exit 1 | |
| fi | |
| TOKEN_JSON=$(kubectl logs pod/curl-token-gen-vm -n ${NAMESPACE}) | |
| kubectl delete pod curl-token-gen-vm -n ${NAMESPACE} --ignore-not-found | |
| BOOTSTRAP_TOKEN=$(echo "${TOKEN_JSON}" | jq -r .token) | |
| if [ -z "${BOOTSTRAP_TOKEN}" ] || [ "${BOOTSTRAP_TOKEN}" = "null" ]; then | |
| echo "ERROR: Generated bootstrap token is empty or invalid!" | |
| echo "Response: ${TOKEN_JSON}" | |
| exit 1 | |
| fi | |
| echo "Successfully generated bootstrap token." | |
| echo "Provisioning GCE VM Router instance..." | |
| # Delete existing VM if it exists to ensure fresh deployment with the new token and image | |
| gcloud compute instances delete sam-router-${ENV_NAME}-vm --zone=${ZONE} --project=${GCP_PROJECT_ID} --quiet || true | |
| # Create a new COS VM instance running the sam-router | |
| gcloud compute instances create sam-router-${ENV_NAME}-vm \ | |
| --project=${GCP_PROJECT_ID} \ | |
| --zone=${ZONE} \ | |
| --machine-type=e2-micro \ | |
| --image-family=cos-stable \ | |
| --image-project=cos-cloud \ | |
| --tags=sam-router \ | |
| --metadata=startup-script='#!/bin/bash | |
| set -euo pipefail | |
| EXTERNAL_IP=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip) | |
| docker run -d \ | |
| --name sam-router \ | |
| --network host \ | |
| --restart always \ | |
| ghcr.io/google/sam-router:'"${IMAGE_TAG}"' \ | |
| --control-plane=https://'"${ENV_NAME}"'.sam-mesh.dev \ | |
| --bootstrap-token='"${BOOTSTRAP_TOKEN}"' \ | |
| --listen=/ip4/0.0.0.0/tcp/4501 \ | |
| --listen=/ip4/0.0.0.0/udp/4501/quic-v1 \ | |
| --external-addr=/ip4/${EXTERNAL_IP}/tcp/4501 \ | |
| --external-addr=/ip4/${EXTERNAL_IP}/udp/4501/quic-v1' | |
| - name: Deploy COP Canary | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| export BANANA_BOT_SCRIPT=$(cat site/content/docs/snippets/banana_bot_playground.py | sed 's/^/ /') | |
| envsubst < .github/k8s/sam-node-cop-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/cop-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || { | |
| echo "Cop Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "cop-canary-${ENV_NAME}" "app=cop-canary-${ENV_NAME}" | |
| exit 1 | |
| } | |
| - name: Deploy SAM Box Canary | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| envsubst < .github/k8s/sam-box-canary-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/box-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || { | |
| echo "sam-box Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "box-canary-${ENV_NAME}" "app=box-canary-${ENV_NAME}" | |
| exit 1 | |
| } | |
| - name: Deploy SAM Node Canary | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| envsubst < .github/k8s/sam-node-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/sam-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || { | |
| echo "Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "sam-canary-${ENV_NAME}" "app=sam-canary-${ENV_NAME}" | |
| echo "Rolling back Canary deployment..." | |
| kubectl rollout undo deployment/sam-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} | |
| kubectl rollout status deployment/sam-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true | |
| exit 1 | |
| } | |
| - name: Deploy OpenClaw Canary | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| VAR_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| export GEMINI_API_KEY="${VAR_GEMINI_API_KEY}" | |
| GATEWAY_TOKEN=$(kubectl get secret openclaw-secret-${ENV_NAME} -n ${CANARY_NAMESPACE} -o jsonpath="{.data.gateway-token}" 2>/dev/null | base64 -d || openssl rand -hex 16) | |
| kubectl create secret generic openclaw-secret-${ENV_NAME} \ | |
| --namespace=${CANARY_NAMESPACE} \ | |
| --from-literal=gateway-token="${GATEWAY_TOKEN}" \ | |
| --from-literal=gemini-api-key="${GEMINI_API_KEY}" \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| envsubst < .github/k8s/sam-node-openclaw-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/openclaw-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=300s || { | |
| echo "OpenClaw Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "openclaw-canary-${ENV_NAME}" "app=openclaw-canary-${ENV_NAME}" | |
| echo "Rolling back OpenClaw Canary deployment..." | |
| kubectl rollout undo deployment/openclaw-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} | |
| kubectl rollout status deployment/openclaw-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true | |
| exit 1 | |
| } | |
| - name: Deploy vLLM Canary | |
| if: vars.ENV_NAME != 'hub' | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| VAR_HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| export HF_TOKEN="${VAR_HF_TOKEN}" | |
| kubectl create secret generic vllm-secret-${ENV_NAME} \ | |
| --namespace=${CANARY_NAMESPACE} \ | |
| --from-literal=hf-token="${HF_TOKEN}" \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| envsubst < .github/k8s/sam-node-vllm-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/vllm-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=300s || { | |
| echo "vLLM Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "vllm-canary-${ENV_NAME}" "app=vllm-canary-${ENV_NAME}" | |
| echo "Rolling back vLLM Canary deployment..." | |
| kubectl rollout undo deployment/vllm-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} | |
| kubectl rollout status deployment/vllm-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true | |
| exit 1 | |
| } | |
| - name: Deploy Everything Canary | |
| env: | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| envsubst < .github/k8s/sam-node-everything-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/everything-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || { | |
| echo "Everything Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "everything-canary-${ENV_NAME}" "app=everything-canary-${ENV_NAME}" | |
| echo "Rolling back Everything Canary deployment..." | |
| kubectl rollout undo deployment/everything-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} | |
| kubectl rollout status deployment/everything-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true | |
| exit 1 | |
| } | |
| - name: Deploy OpenRouter Node to Testnet | |
| env: | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| VAR_ENV_NAME: ${{ vars.ENV_NAME }} | |
| VAR_IMAGE_TAG: ${{ env.IMAGE_TAG }} | |
| run: | | |
| print_rollout_diagnostics() { | |
| local namespace="$1" | |
| local deployment="$2" | |
| local selector="$3" | |
| echo "Collecting diagnostics for deployment/${deployment} in namespace ${namespace}..." | |
| kubectl describe deployment/${deployment} -n ${namespace} || true | |
| kubectl get pods -n ${namespace} -l "${selector}" -o wide || true | |
| kubectl describe pods -n ${namespace} -l "${selector}" || true | |
| kubectl get events -n ${namespace} --sort-by=.lastTimestamp || true | |
| for pod in $(kubectl get pods -n ${namespace} -l "${selector}" -o name 2>/dev/null); do | |
| echo "==== Describe ${pod} ====" | |
| kubectl describe -n ${namespace} "${pod}" || true | |
| for container in $(kubectl get -n ${namespace} "${pod}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null); do | |
| echo "==== Logs for ${pod} container ${container} ====" | |
| kubectl logs -n ${namespace} "${pod#pod/}" -c "${container}" --tail=-1 || true | |
| done | |
| done | |
| } | |
| export ENV_NAME="${VAR_ENV_NAME}" | |
| export NAMESPACE="sam-${ENV_NAME}" | |
| export CANARY_NAMESPACE="sam-canary-${ENV_NAME}" | |
| export IMAGE_TAG="${VAR_IMAGE_TAG}" | |
| # Substitute the API key and other variables into the template and apply it to the cluster | |
| envsubst < .github/k8s/sam-node-openrouter-template.yaml | kubectl apply -f - | |
| kubectl rollout status deployment/openrouter-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} --timeout=120s || { | |
| echo "OpenRouter Canary Deployment failed!" | |
| print_rollout_diagnostics "${CANARY_NAMESPACE}" "openrouter-canary-${ENV_NAME}" "app=openrouter-canary-${ENV_NAME}" | |
| echo "Rolling back OpenRouter Canary deployment..." | |
| kubectl rollout undo deployment/openrouter-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} | |
| kubectl rollout status deployment/openrouter-canary-${ENV_NAME} -n ${CANARY_NAMESPACE} || true | |
| exit 1 | |
| } |