diff --git a/.github/workflows/deploy-tee-cpu.yml b/.github/workflows/deploy-tee-cpu.yml new file mode 100644 index 0000000..7ebb58d --- /dev/null +++ b/.github/workflows/deploy-tee-cpu.yml @@ -0,0 +1,301 @@ +# ============================================================================= +# TrustedGenAi - Deploy CPU TEE Infrastructure (Intel TDX) +# ============================================================================= +# +# Deploys Intel TDX confidential VMs to Azure with DeepSeek models. +# Endpoint: tee.vibebrowser.app +# Cost: ~$216/month (Standard_DC4as_v5) +# +# Secrets required: +# - AZURE_CREDENTIALS: Azure service principal JSON +# - AZURE_SUBSCRIPTION_ID: Azure subscription ID +# - TEE_API_KEY: API key for TEE LiteLLM endpoint +# - CLOUDFLARE_TUNNEL_TOKEN_CPU: Cloudflare tunnel token for tee.vibebrowser.app +# +# ============================================================================= + +name: Deploy CPU TEE (Intel TDX) + +on: + workflow_dispatch: + inputs: + environment: + description: 'Deployment environment' + required: true + default: 'dev' + type: choice + options: + - dev + - prod + action: + description: 'Terraform action' + required: true + default: 'plan' + type: choice + options: + - plan + - apply + - destroy + +env: + TF_VERSION: '1.5.0' + TF_WORKING_DIR: './terraform' + DEPLOYMENT_TYPE: 'cpu' + ENDPOINT_DNS: 'tee.vibebrowser.app' + +jobs: + validate: + name: Validate Terraform + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Terraform Format Check + run: terraform fmt -check -recursive + working-directory: ${{ env.TF_WORKING_DIR }} + + - name: Terraform Init + run: terraform init -backend=false + working-directory: ${{ env.TF_WORKING_DIR }} + + - name: Terraform Validate + run: terraform validate + working-directory: ${{ env.TF_WORKING_DIR }} + + plan: + name: Plan CPU TEE + runs-on: ubuntu-latest + needs: validate + if: github.event.inputs.action == 'plan' + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Azure Login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Terraform Init + run: | + terraform init \ + -backend-config="resource_group_name=trustedgenai-tfstate-rg" \ + -backend-config="storage_account_name=trustedgenaitfstate" \ + -backend-config="container_name=tfstate" \ + -backend-config="key=${{ github.event.inputs.environment }}-cpu.terraform.tfstate" + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Terraform Plan + id: plan + run: | + terraform plan \ + -var="azure_subscription_id=${{ secrets.AZURE_SUBSCRIPTION_ID }}" \ + -var="tee_api_key=${{ secrets.TEE_API_KEY }}" \ + -var="enable_cpu_tee=true" \ + -var="enable_gpu_tee=false" \ + -var="cloudflare_tunnel_token=${{ secrets.CLOUDFLARE_TUNNEL_TOKEN_CPU }}" \ + -var="endpoint_dns=${{ env.ENDPOINT_DNS }}" \ + -out=tfplan \ + -no-color + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Upload Plan + uses: actions/upload-artifact@v4 + with: + name: tfplan-${{ github.event.inputs.environment }}-cpu + path: ${{ env.TF_WORKING_DIR }}/tfplan + + - name: Plan Summary + run: | + echo "## CPU TEE Plan Complete" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Environment | ${{ github.event.inputs.environment }} |" >> $GITHUB_STEP_SUMMARY + echo "| VM Size | Standard_DC4as_v5 (Intel TDX) |" >> $GITHUB_STEP_SUMMARY + echo "| Estimated Cost | ~\$216/month |" >> $GITHUB_STEP_SUMMARY + echo "| Endpoint | https://${{ env.ENDPOINT_DNS }} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Run with \`action: apply\` to deploy." >> $GITHUB_STEP_SUMMARY + + deploy: + name: Deploy CPU TEE + runs-on: ubuntu-latest + needs: validate + if: github.event.inputs.action == 'apply' + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Azure Login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Terraform Init + run: | + terraform init \ + -backend-config="resource_group_name=trustedgenai-tfstate-rg" \ + -backend-config="storage_account_name=trustedgenaitfstate" \ + -backend-config="container_name=tfstate" \ + -backend-config="key=${{ github.event.inputs.environment }}-cpu.terraform.tfstate" + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Terraform Apply + run: | + terraform apply \ + -var="azure_subscription_id=${{ secrets.AZURE_SUBSCRIPTION_ID }}" \ + -var="tee_api_key=${{ secrets.TEE_API_KEY }}" \ + -var="enable_cpu_tee=true" \ + -var="enable_gpu_tee=false" \ + -var="cloudflare_tunnel_token=${{ secrets.CLOUDFLARE_TUNNEL_TOKEN_CPU }}" \ + -var="endpoint_dns=${{ env.ENDPOINT_DNS }}" \ + -auto-approve + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Get Outputs + id: outputs + run: | + echo "public_ip=$(terraform output -raw cpu_tee_public_ip 2>/dev/null || echo 'N/A')" >> $GITHUB_OUTPUT + echo "ssh_command=$(terraform output -raw cpu_tee_ssh_command 2>/dev/null || echo 'N/A')" >> $GITHUB_OUTPUT + working-directory: ${{ env.TF_WORKING_DIR }} + + - name: Wait for VM to be ready + run: | + echo "Waiting 120 seconds for VM to initialize..." + sleep 120 + + - name: Verify TEE Attestation + run: | + echo "Checking attestation endpoint..." + curl -sf --retry 5 --retry-delay 30 "https://${{ env.ENDPOINT_DNS }}/v1/attestation" | jq '{platform, tee_verified, vm_size}' || echo "Attestation endpoint not yet available" + + - name: Verify LiteLLM Health + run: | + echo "Checking LiteLLM health..." + curl -sf --retry 5 --retry-delay 30 "https://${{ env.ENDPOINT_DNS }}/health" || echo "Health endpoint not yet available" + + - name: Deployment Summary + run: | + echo "## CPU TEE Deployment Complete" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Environment | ${{ github.event.inputs.environment }} |" >> $GITHUB_STEP_SUMMARY + echo "| VM Size | Standard_DC4as_v5 (Intel TDX) |" >> $GITHUB_STEP_SUMMARY + echo "| Public IP | ${{ steps.outputs.outputs.public_ip }} |" >> $GITHUB_STEP_SUMMARY + echo "| SSH | ${{ steps.outputs.outputs.ssh_command }} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Endpoints" >> $GITHUB_STEP_SUMMARY + echo "- Attestation: https://${{ env.ENDPOINT_DNS }}/v1/attestation" >> $GITHUB_STEP_SUMMARY + echo "- LiteLLM API: https://${{ env.ENDPOINT_DNS }}/v1" >> $GITHUB_STEP_SUMMARY + echo "- Health: https://${{ env.ENDPOINT_DNS }}/health" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Models Available" >> $GITHUB_STEP_SUMMARY + echo "- \`deepseek-r1-tee\` (1.5B distill, CPU inference)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Billing Integration" >> $GITHUB_STEP_SUMMARY + echo "TEE models available to Max tier subscribers via api.vibebrowser.app" >> $GITHUB_STEP_SUMMARY + + destroy: + name: Destroy CPU TEE + runs-on: ubuntu-latest + needs: validate + if: github.event.inputs.action == 'destroy' + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Azure Login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Terraform Init + run: | + terraform init \ + -backend-config="resource_group_name=trustedgenai-tfstate-rg" \ + -backend-config="storage_account_name=trustedgenaitfstate" \ + -backend-config="container_name=tfstate" \ + -backend-config="key=${{ github.event.inputs.environment }}-cpu.terraform.tfstate" + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Terraform Destroy + run: | + terraform destroy \ + -var="azure_subscription_id=${{ secrets.AZURE_SUBSCRIPTION_ID }}" \ + -var="tee_api_key=${{ secrets.TEE_API_KEY }}" \ + -var="enable_cpu_tee=true" \ + -var="enable_gpu_tee=false" \ + -var="cloudflare_tunnel_token=${{ secrets.CLOUDFLARE_TUNNEL_TOKEN_CPU }}" \ + -var="endpoint_dns=${{ env.ENDPOINT_DNS }}" \ + -auto-approve + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Destroy Summary + run: | + echo "## CPU TEE Infrastructure Destroyed" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Environment | ${{ github.event.inputs.environment }} |" >> $GITHUB_STEP_SUMMARY + echo "| Endpoint | ${{ env.ENDPOINT_DNS }} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Monthly cost savings: ~\$216/month" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/deploy-tee-gpu.yml b/.github/workflows/deploy-tee-gpu.yml new file mode 100644 index 0000000..4d3d3fa --- /dev/null +++ b/.github/workflows/deploy-tee-gpu.yml @@ -0,0 +1,350 @@ +# ============================================================================= +# TrustedGenAi - Deploy GPU TEE Infrastructure (NVIDIA H100 CC) +# ============================================================================= +# +# Deploys NVIDIA H100 Confidential Computing VMs to Azure with DeepSeek models. +# Endpoint: tee-gpu.vibebrowser.app +# Cost: ~$6,300/month (NCCads_H100_v5) +# +# WARNING: This is expensive infrastructure. Use sparingly. +# +# Secrets required: +# - AZURE_CREDENTIALS: Azure service principal JSON +# - AZURE_SUBSCRIPTION_ID: Azure subscription ID +# - TEE_API_KEY: API key for TEE LiteLLM endpoint +# - CLOUDFLARE_TUNNEL_TOKEN_GPU: Cloudflare tunnel token for tee-gpu.vibebrowser.app +# +# ============================================================================= + +name: Deploy GPU TEE (NVIDIA H100) + +on: + workflow_dispatch: + inputs: + environment: + description: 'Deployment environment' + required: true + default: 'dev' + type: choice + options: + - dev + - prod + action: + description: 'Terraform action' + required: true + default: 'plan' + type: choice + options: + - plan + - apply + - destroy + model_size: + description: 'DeepSeek model to deploy' + required: true + default: '7b' + type: choice + options: + - 7b # DeepSeek-R1-Distill-Qwen-7B (~14GB VRAM) + - 14b # DeepSeek-R1-Distill-Qwen-14B (~28GB VRAM) + - 32b # DeepSeek-R1-Distill-Qwen-32B (~64GB VRAM) + - 70b # DeepSeek-R1-Distill-Llama-70B (80GB VRAM, full H100) + +env: + TF_VERSION: '1.5.0' + TF_WORKING_DIR: './terraform' + DEPLOYMENT_TYPE: 'gpu' + ENDPOINT_DNS: 'tee-gpu.vibebrowser.app' + +jobs: + validate: + name: Validate Terraform + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Terraform Format Check + run: terraform fmt -check -recursive + working-directory: ${{ env.TF_WORKING_DIR }} + + - name: Terraform Init + run: terraform init -backend=false + working-directory: ${{ env.TF_WORKING_DIR }} + + - name: Terraform Validate + run: terraform validate + working-directory: ${{ env.TF_WORKING_DIR }} + + cost-warning: + name: Cost Warning + runs-on: ubuntu-latest + if: github.event.inputs.action == 'apply' + steps: + - name: Display Cost Warning + run: | + echo "::warning::GPU TEE deployment costs ~\$6,300/month (~\$210/day)" + echo "::warning::Make sure to destroy when not in use" + echo "" + echo "## Cost Warning" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY + echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Monthly Cost | ~\$6,300 |" >> $GITHUB_STEP_SUMMARY + echo "| Daily Cost | ~\$210 |" >> $GITHUB_STEP_SUMMARY + echo "| Hourly Cost | ~\$8.75 |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Remember to run \`action: destroy\` when done testing." >> $GITHUB_STEP_SUMMARY + + plan: + name: Plan GPU TEE (${{ github.event.inputs.model_size }}) + runs-on: ubuntu-latest + needs: validate + if: github.event.inputs.action == 'plan' + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Azure Login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Terraform Init + run: | + terraform init \ + -backend-config="resource_group_name=trustedgenai-tfstate-rg" \ + -backend-config="storage_account_name=trustedgenaitfstate" \ + -backend-config="container_name=tfstate" \ + -backend-config="key=${{ github.event.inputs.environment }}-gpu.terraform.tfstate" + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Terraform Plan + id: plan + run: | + terraform plan \ + -var="azure_subscription_id=${{ secrets.AZURE_SUBSCRIPTION_ID }}" \ + -var="tee_api_key=${{ secrets.TEE_API_KEY }}" \ + -var="enable_cpu_tee=false" \ + -var="enable_gpu_tee=true" \ + -var="gpu_model_size=${{ github.event.inputs.model_size }}" \ + -var="cloudflare_tunnel_token=${{ secrets.CLOUDFLARE_TUNNEL_TOKEN_GPU }}" \ + -var="endpoint_dns=${{ env.ENDPOINT_DNS }}" \ + -out=tfplan \ + -no-color + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Upload Plan + uses: actions/upload-artifact@v4 + with: + name: tfplan-${{ github.event.inputs.environment }}-gpu-${{ github.event.inputs.model_size }} + path: ${{ env.TF_WORKING_DIR }}/tfplan + + - name: Plan Summary + run: | + echo "## GPU TEE Plan Complete" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Environment | ${{ github.event.inputs.environment }} |" >> $GITHUB_STEP_SUMMARY + echo "| VM Size | NCCads_H100_v5 (NVIDIA H100 CC) |" >> $GITHUB_STEP_SUMMARY + echo "| Model Size | ${{ github.event.inputs.model_size }} |" >> $GITHUB_STEP_SUMMARY + echo "| Estimated Cost | ~\$6,300/month |" >> $GITHUB_STEP_SUMMARY + echo "| Endpoint | https://${{ env.ENDPOINT_DNS }} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Run with \`action: apply\` to deploy." >> $GITHUB_STEP_SUMMARY + + deploy: + name: Deploy GPU TEE (${{ github.event.inputs.model_size }}) + runs-on: ubuntu-latest + needs: [validate, cost-warning] + if: github.event.inputs.action == 'apply' + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Azure Login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Terraform Init + run: | + terraform init \ + -backend-config="resource_group_name=trustedgenai-tfstate-rg" \ + -backend-config="storage_account_name=trustedgenaitfstate" \ + -backend-config="container_name=tfstate" \ + -backend-config="key=${{ github.event.inputs.environment }}-gpu.terraform.tfstate" + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Terraform Apply + run: | + terraform apply \ + -var="azure_subscription_id=${{ secrets.AZURE_SUBSCRIPTION_ID }}" \ + -var="tee_api_key=${{ secrets.TEE_API_KEY }}" \ + -var="enable_cpu_tee=false" \ + -var="enable_gpu_tee=true" \ + -var="gpu_model_size=${{ github.event.inputs.model_size }}" \ + -var="cloudflare_tunnel_token=${{ secrets.CLOUDFLARE_TUNNEL_TOKEN_GPU }}" \ + -var="endpoint_dns=${{ env.ENDPOINT_DNS }}" \ + -auto-approve + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Get Outputs + id: outputs + run: | + echo "public_ip=$(terraform output -raw gpu_tee_public_ip 2>/dev/null || echo 'N/A')" >> $GITHUB_OUTPUT + echo "ssh_command=$(terraform output -raw gpu_tee_ssh_command 2>/dev/null || echo 'N/A')" >> $GITHUB_OUTPUT + working-directory: ${{ env.TF_WORKING_DIR }} + + - name: Wait for VM to be ready + run: | + echo "Waiting 180 seconds for GPU VM to initialize and load model..." + sleep 180 + + - name: Verify TEE Attestation + run: | + echo "Checking GPU TEE attestation endpoint..." + curl -sf --retry 5 --retry-delay 30 "https://${{ env.ENDPOINT_DNS }}/v1/attestation" | jq '{platform, tee_verified, gpu_confidential, vm_size}' || echo "Attestation endpoint not yet available" + + - name: Verify LiteLLM Health + run: | + echo "Checking LiteLLM health..." + curl -sf --retry 5 --retry-delay 30 "https://${{ env.ENDPOINT_DNS }}/health" || echo "Health endpoint not yet available" + + - name: Verify Model Loaded + run: | + echo "Checking available models..." + curl -sf --retry 3 --retry-delay 30 "https://${{ env.ENDPOINT_DNS }}/v1/models" -H "Authorization: Bearer ${{ secrets.TEE_API_KEY }}" | jq '.data[].id' || echo "Models endpoint not yet available" + + - name: Deployment Summary + run: | + MODEL_NAME="deepseek-r1-distill-${{ github.event.inputs.model_size }}-tee" + + echo "## GPU TEE Deployment Complete" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Environment | ${{ github.event.inputs.environment }} |" >> $GITHUB_STEP_SUMMARY + echo "| VM Size | NCCads_H100_v5 (80GB HBM3) |" >> $GITHUB_STEP_SUMMARY + echo "| Model | ${{ github.event.inputs.model_size }} |" >> $GITHUB_STEP_SUMMARY + echo "| Public IP | ${{ steps.outputs.outputs.public_ip }} |" >> $GITHUB_STEP_SUMMARY + echo "| SSH | ${{ steps.outputs.outputs.ssh_command }} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Endpoints" >> $GITHUB_STEP_SUMMARY + echo "- Attestation: https://${{ env.ENDPOINT_DNS }}/v1/attestation" >> $GITHUB_STEP_SUMMARY + echo "- LiteLLM API: https://${{ env.ENDPOINT_DNS }}/v1" >> $GITHUB_STEP_SUMMARY + echo "- Health: https://${{ env.ENDPOINT_DNS }}/health" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Models Available" >> $GITHUB_STEP_SUMMARY + echo "- \`${MODEL_NAME}\` (GPU accelerated, H100 CC)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Billing Integration" >> $GITHUB_STEP_SUMMARY + echo "GPU TEE models available to Max tier subscribers via api.vibebrowser.app" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Cost Reminder" >> $GITHUB_STEP_SUMMARY + echo "This deployment costs ~\$210/day. Run \`action: destroy\` when done." >> $GITHUB_STEP_SUMMARY + + destroy: + name: Destroy GPU TEE + runs-on: ubuntu-latest + needs: validate + if: github.event.inputs.action == 'destroy' + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Azure Login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Terraform Init + run: | + terraform init \ + -backend-config="resource_group_name=trustedgenai-tfstate-rg" \ + -backend-config="storage_account_name=trustedgenaitfstate" \ + -backend-config="container_name=tfstate" \ + -backend-config="key=${{ github.event.inputs.environment }}-gpu.terraform.tfstate" + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Terraform Destroy + run: | + terraform destroy \ + -var="azure_subscription_id=${{ secrets.AZURE_SUBSCRIPTION_ID }}" \ + -var="tee_api_key=${{ secrets.TEE_API_KEY }}" \ + -var="enable_cpu_tee=false" \ + -var="enable_gpu_tee=true" \ + -var="gpu_model_size=7b" \ + -var="cloudflare_tunnel_token=${{ secrets.CLOUDFLARE_TUNNEL_TOKEN_GPU }}" \ + -var="endpoint_dns=${{ env.ENDPOINT_DNS }}" \ + -auto-approve + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Destroy Summary + run: | + echo "## GPU TEE Infrastructure Destroyed" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Environment | ${{ github.event.inputs.environment }} |" >> $GITHUB_STEP_SUMMARY + echo "| Endpoint | ${{ env.ENDPOINT_DNS }} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Monthly cost savings: ~\$6,300/month" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "GPU TEE resources have been destroyed." >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/deploy-tee.yml b/.github/workflows/deploy-tee.yml new file mode 100644 index 0000000..68fccf8 --- /dev/null +++ b/.github/workflows/deploy-tee.yml @@ -0,0 +1,342 @@ +# ============================================================================= +# TrustedGenAi - Deploy TEE Infrastructure +# ============================================================================= +# +# Deploys TEE VMs to Azure with DeepSeek models and connects to VibeBrowser +# billing infrastructure at api.vibebrowser.app. +# +# Triggers: +# - Manual dispatch with environment selection +# - Push to main (plan only) +# +# Secrets required: +# - AZURE_CREDENTIALS: Azure service principal JSON +# - AZURE_SUBSCRIPTION_ID: Azure subscription ID +# - TEE_API_KEY: API key for TEE LiteLLM endpoint +# - CLOUDFLARE_TUNNEL_TOKEN: Cloudflare tunnel token for HTTPS +# +# ============================================================================= + +name: Deploy TEE Infrastructure + +on: + workflow_dispatch: + inputs: + environment: + description: 'Deployment environment' + required: true + default: 'dev' + type: choice + options: + - dev + - prod + deployment_type: + description: 'TEE deployment type' + required: true + default: 'cpu' + type: choice + options: + - cpu # Intel TDX (~$216/month) + - gpu # NVIDIA H100 CC (~$6,300/month) + action: + description: 'Terraform action' + required: true + default: 'plan' + type: choice + options: + - plan + - apply + - destroy + push: + branches: + - main + paths: + - 'terraform/**' + - '.github/workflows/deploy-tee.yml' + +env: + TF_VERSION: '1.5.0' + TF_WORKING_DIR: './terraform' + +jobs: + validate: + name: Validate Terraform + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Terraform Format Check + run: terraform fmt -check -recursive + working-directory: ${{ env.TF_WORKING_DIR }} + + - name: Terraform Init + run: terraform init -backend=false + working-directory: ${{ env.TF_WORKING_DIR }} + + - name: Terraform Validate + run: terraform validate + working-directory: ${{ env.TF_WORKING_DIR }} + + plan: + name: Plan ${{ github.event.inputs.deployment_type || 'cpu' }} TEE + runs-on: ubuntu-latest + needs: validate + if: github.event_name == 'push' || github.event.inputs.action == 'plan' + environment: ${{ github.event.inputs.environment || 'dev' }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Azure Login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Terraform Init + run: | + terraform init \ + -backend-config="resource_group_name=trustedgenai-tfstate-rg" \ + -backend-config="storage_account_name=trustedgenaitfstate" \ + -backend-config="container_name=tfstate" \ + -backend-config="key=${{ github.event.inputs.environment || 'dev' }}.terraform.tfstate" + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Terraform Plan + id: plan + run: | + DEPLOY_TYPE="${{ github.event.inputs.deployment_type || 'cpu' }}" + + if [ "$DEPLOY_TYPE" = "cpu" ]; then + VAR_FLAG="-var=enable_cpu_tee=true" + else + VAR_FLAG="-var=enable_gpu_tee=true" + fi + + terraform plan \ + -var="azure_subscription_id=${{ secrets.AZURE_SUBSCRIPTION_ID }}" \ + -var="tee_api_key=${{ secrets.TEE_API_KEY }}" \ + $VAR_FLAG \ + -out=tfplan \ + -no-color + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Upload Plan + uses: actions/upload-artifact@v4 + with: + name: tfplan-${{ github.event.inputs.environment || 'dev' }}-${{ github.event.inputs.deployment_type || 'cpu' }} + path: ${{ env.TF_WORKING_DIR }}/tfplan + + - name: Comment Plan on PR + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const output = `#### Terraform Plan 📖 + + **Environment:** \`${{ github.event.inputs.environment || 'dev' }}\` + **Deployment Type:** \`${{ github.event.inputs.deployment_type || 'cpu' }}\` + +
Show Plan + + \`\`\` + ${{ steps.plan.outputs.stdout }} + \`\`\` + +
+ + *Pushed by: @${{ github.actor }}*`; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }); + + deploy: + name: Deploy ${{ github.event.inputs.deployment_type }} TEE + runs-on: ubuntu-latest + needs: plan + if: github.event.inputs.action == 'apply' + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Azure Login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Download Plan + uses: actions/download-artifact@v4 + with: + name: tfplan-${{ github.event.inputs.environment }}-${{ github.event.inputs.deployment_type }} + path: ${{ env.TF_WORKING_DIR }} + + - name: Terraform Init + run: | + terraform init \ + -backend-config="resource_group_name=trustedgenai-tfstate-rg" \ + -backend-config="storage_account_name=trustedgenaitfstate" \ + -backend-config="container_name=tfstate" \ + -backend-config="key=${{ github.event.inputs.environment }}.terraform.tfstate" + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Terraform Apply + run: terraform apply -auto-approve tfplan + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Get Outputs + id: outputs + run: | + echo "public_ip=$(terraform output -raw deepseek_confidential_public_ip 2>/dev/null || echo 'N/A')" >> $GITHUB_OUTPUT + echo "ssh_command=$(terraform output -raw deepseek_confidential_ssh_command 2>/dev/null || echo 'N/A')" >> $GITHUB_OUTPUT + working-directory: ${{ env.TF_WORKING_DIR }} + + - name: Wait for VM to be ready + run: | + echo "Waiting 120 seconds for VM to initialize..." + sleep 120 + + - name: Verify TEE Attestation + run: | + PUBLIC_IP="${{ steps.outputs.outputs.public_ip }}" + if [ "$PUBLIC_IP" != "N/A" ]; then + echo "Checking attestation endpoint..." + curl -s --retry 5 --retry-delay 30 "http://${PUBLIC_IP}:4001/v1/attestation" | jq '{platform, tee_verified, vm_size}' + fi + + - name: Configure Cloudflare Tunnel + if: success() + run: | + echo "Cloudflare tunnel configuration would be applied here" + echo "Tunnel token stored in secrets.CLOUDFLARE_TUNNEL_TOKEN" + # The tunnel is configured via cloud-init on the VM + # This step verifies the tunnel is active + + - name: Deployment Summary + run: | + echo "## TEE Deployment Complete" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Environment | ${{ github.event.inputs.environment }} |" >> $GITHUB_STEP_SUMMARY + echo "| Type | ${{ github.event.inputs.deployment_type }} |" >> $GITHUB_STEP_SUMMARY + echo "| Public IP | ${{ steps.outputs.outputs.public_ip }} |" >> $GITHUB_STEP_SUMMARY + echo "| SSH | ${{ steps.outputs.outputs.ssh_command }} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Endpoints" >> $GITHUB_STEP_SUMMARY + echo "- Attestation: https://tee.vibebrowser.app/attestation" >> $GITHUB_STEP_SUMMARY + echo "- LiteLLM API: https://tee.vibebrowser.app/v1" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Billing Integration" >> $GITHUB_STEP_SUMMARY + echo "TEE model \`deepseek-r1-tee\` is available to Max tier subscribers via api.vibebrowser.app" >> $GITHUB_STEP_SUMMARY + + destroy: + name: Destroy ${{ github.event.inputs.deployment_type }} TEE + runs-on: ubuntu-latest + needs: validate + if: github.event.inputs.action == 'destroy' + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Azure Login + uses: azure/login@v2 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Terraform Init + run: | + terraform init \ + -backend-config="resource_group_name=trustedgenai-tfstate-rg" \ + -backend-config="storage_account_name=trustedgenaitfstate" \ + -backend-config="container_name=tfstate" \ + -backend-config="key=${{ github.event.inputs.environment }}.terraform.tfstate" + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Terraform Destroy + run: | + DEPLOY_TYPE="${{ github.event.inputs.deployment_type }}" + + if [ "$DEPLOY_TYPE" = "cpu" ]; then + VAR_FLAG="-var=enable_cpu_tee=true" + else + VAR_FLAG="-var=enable_gpu_tee=true" + fi + + terraform destroy \ + -var="azure_subscription_id=${{ secrets.AZURE_SUBSCRIPTION_ID }}" \ + -var="tee_api_key=${{ secrets.TEE_API_KEY }}" \ + $VAR_FLAG \ + -auto-approve + working-directory: ${{ env.TF_WORKING_DIR }} + env: + ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + + - name: Destroy Summary + run: | + echo "## TEE Infrastructure Destroyed" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Environment | ${{ github.event.inputs.environment }} |" >> $GITHUB_STEP_SUMMARY + echo "| Type | ${{ github.event.inputs.deployment_type }} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Resources have been destroyed. Monthly cost savings:" >> $GITHUB_STEP_SUMMARY + echo "- CPU TEE: ~\$216/month" >> $GITHUB_STEP_SUMMARY + echo "- GPU TEE: ~\$6,300/month" >> $GITHUB_STEP_SUMMARY diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c2323b1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Terraform +.terraform/ +*.tfstate +*.tfstate.backup +*.tfvars +.terraform.lock.hcl + +# Secrets +*.pem +*.key +secrets/ + +# OS +.DS_Store + +# IDE +.idea/ +.vscode/ + +# Build artifacts +*.log diff --git a/arxiv-submission/main.tex b/arxiv-submission/main.tex new file mode 100644 index 0000000..53ca1b3 --- /dev/null +++ b/arxiv-submission/main.tex @@ -0,0 +1,433 @@ +\documentclass[11pt,a4paper]{article} + +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage{amsmath,amssymb} +\usepackage{graphicx} +\usepackage{hyperref} +\usepackage{listings} +\usepackage{xcolor} +\usepackage{booktabs} +\usepackage{geometry} +\usepackage{float} +\usepackage{enumitem} +\usepackage{authblk} + +\geometry{margin=1in} + +\lstset{ + basicstyle=\ttfamily\small, + breaklines=true, + frame=single, + backgroundcolor=\color{gray!10}, + numbers=left, + numberstyle=\tiny\color{gray}, +} + +\hypersetup{ + colorlinks=true, + linkcolor=blue, + citecolor=blue, + urlcolor=blue +} + +\title{TrustedGenAi: Privacy-Preserving LLM Inference with\\Hardware-Attested Trusted Execution Environments} + +\author[1]{Dzianis Vashchuk} +\author[2]{Claude-Opus-4.5\thanks{AI research assistant. Contributions include infrastructure implementation, documentation, and code generation.}} +\affil[1]{VibeTechnologies, \texttt{dzianis\_v@pm.me}} +\affil[2]{Anthropic} + +\date{January 2026} + +\begin{document} + +\maketitle + +\begin{abstract} +We present TrustedGenAi, an open-source infrastructure for deploying Large Language Model (LLM) inference within Trusted Execution Environments (TEEs) with cryptographic remote attestation. Our implementation runs self-hosted DeepSeek models on Azure Confidential VMs with Intel TDX, providing hardware-enforced memory encryption and verifiable privacy guarantees. We introduce a remote attestation API that enables clients to cryptographically verify TEE execution before submitting sensitive prompts. Our production deployment demonstrates practical feasibility with 12 tokens/second on CPU TEE and projects 150+ tokens/second on GPU TEE with NVIDIA H100 Confidential Computing. The complete infrastructure, including Terraform configurations and attestation services, is available at \url{https://github.com/VibeTechnologies/TrustedGenAi}. +\end{abstract} + +\section{Introduction} + +The widespread adoption of Large Language Models (LLMs) for sensitive applications---including browser automation, code generation, and document analysis---raises fundamental privacy concerns. Users must trust that their prompts are not logged, their data is not used for training, and their credentials remain confidential. Current cloud-hosted LLM APIs provide no cryptographic guarantees about data handling; users rely entirely on provider policies and legal agreements. + +Trusted Execution Environments (TEEs) offer a hardware-based solution by providing isolated execution contexts where data remains encrypted even from the cloud operator. However, deploying production LLM inference within TEEs presents unique challenges: model size constraints, performance overhead, and the complexity of remote attestation for end users. + +\subsection{Contributions} + +We make the following contributions: + +\begin{enumerate} + \item \textbf{Production TEE-LLM Infrastructure}: We deploy and validate an end-to-end LLM inference system on Azure Confidential VMs with Intel TDX, demonstrating practical feasibility for privacy-critical workloads. + + \item \textbf{Remote Attestation API}: We implement a REST API that provides cryptographic proof of TEE execution, enabling clients to verify hardware isolation before submitting sensitive data. + + \item \textbf{Open-Source Reference Implementation}: We release complete infrastructure code, including Terraform configurations, attestation services, and client integration examples. + + \item \textbf{Performance Benchmarks}: We provide empirical measurements of inference performance on both CPU TEE (Intel TDX) and projections for GPU TEE (NVIDIA H100 Confidential Computing). +\end{enumerate} + +\section{Background} + +\subsection{Trusted Execution Environments} + +A Trusted Execution Environment is a secure, isolated processing environment that guarantees confidentiality and integrity of code and data. The Confidential Computing Consortium defines TEEs as hardware-based, attested environments that protect data in use \cite{ccc}. + +\subsubsection{Intel Trust Domain Extensions (TDX)} + +Intel TDX \cite{tdx} provides hardware-isolated virtual machines called Trust Domains with the following properties: + +\begin{itemize} + \item Memory encryption using CPU-managed keys (AES-256-XTS) + \item Isolation from hypervisor, other VMs, and host OS + \item Hardware-rooted remote attestation + \item Minimal performance overhead (typically $<$5\% for compute-bound workloads) +\end{itemize} + +Azure offers Intel TDX via the DCesv5 VM series (e.g., Standard\_DC4es\_v5). + +\subsubsection{AMD Secure Encrypted Virtualization (SEV-SNP)} + +AMD SEV-SNP \cite{sev} provides similar guarantees with: + +\begin{itemize} + \item Per-VM encryption keys managed by AMD Secure Processor + \item Secure Nested Paging preventing hypervisor memory remapping attacks + \item No guest modifications required (transparent to applications) +\end{itemize} + +Azure offers AMD SEV-SNP via the DCasv5 series and NCCads\_H100\_v5 for GPU workloads. + +\subsection{Remote Attestation} + +Remote attestation enables a client to cryptographically verify that code is running within a genuine TEE. The attestation flow consists of: + +\begin{enumerate} + \item TEE generates a hardware-signed attestation report containing platform measurements + \item Report is signed by manufacturer (Intel/AMD) or cloud provider (Azure) + \item Client verifies signature chain and platform configuration + \item If valid, client trusts subsequent interactions +\end{enumerate} + +Azure Attestation provides PKCS7-signed documents containing VM identity, TPM Platform Configuration Register (PCR) values, and TEE activation proof. + +\section{Threat Model} + +We consider an adversary with the following capabilities: + +\begin{itemize} + \item \textbf{Malicious cloud operator}: Full control over hypervisor, physical access to hardware, ability to inspect VM memory in standard deployments + \item \textbf{Compromised service operator}: Access to application deployment, configuration, and logs + \item \textbf{Network adversary}: Ability to intercept and modify network traffic (mitigated by TLS) +\end{itemize} + +\textbf{Out of scope}: Side-channel attacks on TEE implementations, supply chain attacks on hardware, and denial-of-service attacks. + +\subsection{Security Goals} + +\begin{enumerate} + \item \textbf{Prompt Confidentiality}: User prompts are never accessible to operators or cloud providers + \item \textbf{Response Integrity}: Model outputs cannot be tampered with by external parties + \item \textbf{Verifiable Execution}: Clients can cryptographically verify TEE deployment + \item \textbf{Operator Blindness}: Service operators cannot access user data +\end{enumerate} + +\section{System Architecture} + +\subsection{Overview} + +TrustedGenAi deploys an OpenAI-compatible LLM API within an Azure Confidential VM. The architecture consists of three components: + +\begin{enumerate} + \item \textbf{LiteLLM Proxy}: OpenAI-compatible API gateway supporting multiple model backends + \item \textbf{Ollama/vLLM}: Local model inference engine running DeepSeek models + \item \textbf{Attestation API}: REST endpoint providing cryptographic TEE verification +\end{enumerate} + +\begin{figure}[H] +\centering +\begin{verbatim} ++----------------------------------------------------------+ +| Client Application | ++---------------------------+------------------------------+ + | + v HTTPS (TLS 1.3) ++----------------------------------------------------------+ +| Cloudflare Edge (TLS Termination) | ++---------------------------+------------------------------+ + | + v Cloudflare Tunnel ++----------------------------------------------------------+ +| Azure Confidential VM (Intel TDX / AMD SEV-SNP) | +| | +| +---------------------+ +---------------------------+ | +| | LiteLLM (port 4000) | | Attestation API (port 4001)| | +| | - OpenAI API | | - Azure PKCS7 proof | | +| | - API key auth | | - TPM PCR values | | +| +---------+-----------+ | - TEE dmesg proof | | +| | +---------------------------+ | +| v | +| +---------------------+ | +| | Ollama (port 11434) | | +| | - DeepSeek-R1 | | +| +---------------------+ | +| | +| [Hardware: Memory Encryption via Intel TDX] | ++----------------------------------------------------------+ +\end{verbatim} +\caption{TrustedGenAi System Architecture} +\end{figure} + +\subsection{Attestation API Design} + +The attestation endpoint returns a JSON document containing multiple verification layers: + +\begin{lstlisting}[language=json,caption={Attestation API Response}] +{ + "platform": "Intel-TDX", + "vm_size": "Standard_DC4es_v5", + "tee_verified": true, + "azure_attestation": { + "encoding": "pkcs7", + "signature": "" + }, + "tpm_pcr_sha256": { + "0": "0x2ADE8023...", + "7": "0xF8C9E2A1..." + }, + "tee_dmesg": [ + "Memory Encryption Features active: Intel TDX" + ] +} +\end{lstlisting} + +\textbf{Verification Layers}: + +\begin{enumerate} + \item \texttt{platform}: TEE technology (Intel-TDX or AMD-SEV-SNP) + \item \texttt{azure\_attestation}: PKCS7 document signed by Microsoft Azure, containing VM identity and timestamp + \item \texttt{tpm\_pcr\_sha256}: TPM Platform Configuration Registers for software integrity verification + \item \texttt{tee\_dmesg}: Linux kernel messages proving TEE activation +\end{enumerate} + +\subsection{Client Verification Flow} + +Clients should verify attestation before submitting sensitive prompts: + +\begin{lstlisting}[language=JavaScript,caption={Client Verification Example}] +async function verifyAndChat(prompt) { + const TEE_API = 'https://tee.vibebrowser.app'; + + // Step 1: Fetch and verify attestation + const attestation = await fetch(`${TEE_API}/attestation`) + .then(r => r.json()); + + if (!attestation.tee_verified) { + throw new Error('TEE verification failed'); + } + + if (!attestation.tee_dmesg.some( + l => l.includes('Intel TDX') || l.includes('SEV-SNP'))) { + throw new Error('TEE not active'); + } + + // Step 2: Submit prompt to verified TEE + return fetch(`${TEE_API}/v1/chat/completions`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': 'Bearer ' + }, + body: JSON.stringify({ + model: 'deepseek-r1', + messages: [{ role: 'user', content: prompt }] + }) + }).then(r => r.json()); +} +\end{lstlisting} + +\section{Implementation} + +\subsection{Infrastructure as Code} + +We provide Terraform configurations for reproducible deployment: + +\begin{lstlisting}[language=bash,caption={Deployment Commands}] +# Clone repository +git clone https://github.com/VibeTechnologies/TrustedGenAi +cd TrustedGenAi/terraform + +# Deploy CPU TEE (Intel TDX) +terraform init +terraform apply -var="enable_cpu_tee=true" + +# Deploy GPU TEE (NVIDIA H100 CC) +terraform apply -var="enable_gpu_tee=true" +\end{lstlisting} + +\subsection{CPU TEE Deployment} + +Our production CPU TEE deployment uses: + +\begin{table}[H] +\centering +\caption{CPU TEE Configuration} +\begin{tabular}{@{}ll@{}} +\toprule +\textbf{Parameter} & \textbf{Value} \\ +\midrule +VM Size & Standard\_DC4es\_v5 \\ +vCPUs & 4 \\ +Memory & 16 GB \\ +TEE Type & Intel TDX \\ +OS & Ubuntu 22.04 LTS (Confidential) \\ +Model & DeepSeek-R1 1.5B \\ +Cost & \$216/month \\ +\bottomrule +\end{tabular} +\end{table} + +\subsection{GPU TEE Configuration (Projected)} + +For production workloads requiring higher throughput, we provide Terraform configurations for NVIDIA H100 Confidential Computing. \textbf{Note: GPU TEE has not been deployed; the following specifications and performance numbers are projections based on hardware capabilities.} + +\begin{table}[H] +\centering +\caption{GPU TEE Configuration (Projected)} +\begin{tabular}{@{}ll@{}} +\toprule +\textbf{Parameter} & \textbf{Value} \\ +\midrule +VM Size & Standard\_NCC40ads\_H100\_v5 \\ +vCPUs & 40 (AMD EPYC Genoa) \\ +Memory & 320 GB \\ +GPU & 1x NVIDIA H100 NVL (94 GB HBM3) \\ +TEE Type & AMD SEV-SNP + NVIDIA CC \\ +Model & DeepSeek-R1-Distill-7B \\ +Cost & \$6,300/month \\ +\bottomrule +\end{tabular} +\end{table} + +\section{Evaluation} + +\subsection{Performance Benchmarks} + +We measured inference performance across deployment configurations: + +\begin{table}[H] +\centering +\caption{Inference Performance by Configuration} +\begin{tabular}{@{}lllll@{}} +\toprule +\textbf{Config} & \textbf{Model} & \textbf{Tokens/s} & \textbf{Latency} & \textbf{Cost/1M tokens} \\ +\midrule +CPU TEE & deepseek-r1:1.5b & 12 & 83ms/tok & \$5.00 \\ +CPU TEE & deepseek-r1:7b & 0.7 & 1.4s/tok & \$85.00 \\ +GPU TEE (proj.) & DeepSeek-R1-7B & 150 & 7ms/tok & \$0.40 \\ +\bottomrule +\end{tabular} +\end{table} + +\subsection{Attestation Latency} + +Attestation API response time: 150-300ms (includes Azure metadata service call). + +\subsection{End-to-End Verification} + +We validated the deployment with integration tests: + +\begin{itemize} + \item \textbf{Backend Tests}: 4/4 passing (attestation, models, chat, health) + \item \textbf{Extension Integration}: 5/5 passing (provider config, HTTPS, wrapper, connectivity, build) + \item \textbf{TEE Verification}: Intel TDX confirmed via \texttt{dmesg} and Azure attestation +\end{itemize} + +\section{Security Analysis} + +\subsection{Trust Comparison} + +\begin{table}[H] +\centering +\caption{Trust Requirements by Deployment Model} +\begin{tabular}{@{}lccc@{}} +\toprule +\textbf{Trust Assumption} & \textbf{Cloud API} & \textbf{Self-Hosted} & \textbf{TEE} \\ +\midrule +Trust cloud infrastructure & Yes & Yes & Hardware-verified \\ +Trust service operator & Yes & Yes & No \\ +Trust model provider & Yes & No & No \\ +Cryptographic verification & No & No & Yes \\ +\bottomrule +\end{tabular} +\end{table} + +\subsection{Limitations} + +\begin{enumerate} + \item \textbf{TLS Termination}: Cloudflare terminates TLS before the TEE. For maximum security, clients should establish TLS directly to the TEE. + + \item \textbf{Side Channels}: TEE implementations may be vulnerable to side-channel attacks \cite{side-channels}. Our threat model excludes these. + + \item \textbf{Model Size Constraints}: CPU TEE limits practical model size to approximately 7B parameters due to memory and performance constraints. + + \item \textbf{Region Availability}: GPU TEE (NCCads\_H100\_v5) is limited to East US 2 and West Europe regions. +\end{enumerate} + +\section{Related Work} + +\textbf{Confidential Computing for ML}: Prior work has explored TEE-based machine learning \cite{slalom, privado}, primarily focusing on training privacy. Our work addresses inference privacy with remote attestation for end users. + +\textbf{Private LLM Inference}: Approaches include differential privacy \cite{dp-llm}, secure multi-party computation \cite{mpc-llm}, and homomorphic encryption \cite{he-llm}. TEEs offer lower latency at the cost of trusting hardware manufacturers. + +\textbf{Decentralized AI}: Projects like Marlin Oyster provide on-chain attestation for TEE workloads. Our work focuses on centralized deployment with client-verifiable attestation. + +\section{Conclusion} + +We presented TrustedGenAi, a production-ready infrastructure for privacy-preserving LLM inference using Trusted Execution Environments. Our implementation demonstrates that TEE-based LLM deployment is practical today, with acceptable performance for many use cases and a clear path to GPU acceleration. + +The key insight is that remote attestation transforms the trust model: instead of trusting service operators, users verify hardware-signed cryptographic proofs. This enables privacy-critical applications that were previously infeasible with cloud-hosted LLMs. + +\textbf{Open Source}: Complete infrastructure code is available at:\\ \url{https://github.com/VibeTechnologies/TrustedGenAi} + +\subsection{Future Work} + +\begin{enumerate} + \item \textbf{Direct TLS to TEE}: Eliminate Cloudflare from the trust path + \item \textbf{On-Chain Attestation}: Publish attestation proofs to blockchain for auditability + \item \textbf{Multi-Party TEE}: Distribute inference across multiple TEE nodes + \item \textbf{Larger Models}: Deploy DeepSeek-V3 on multi-GPU TEE clusters +\end{enumerate} + +\begin{thebibliography}{99} + +\bibitem{ccc} Confidential Computing Consortium. \textit{A Technical Analysis of Confidential Computing}. 2022. \url{https://confidentialcomputing.io/} + +\bibitem{tdx} Intel Corporation. \textit{Intel Trust Domain Extensions (Intel TDX) Module Architecture}. 2023. \url{https://www.intel.com/content/www/us/en/developer/tools/trust-domain-extensions/documentation.html} + +\bibitem{sev} AMD. \textit{AMD SEV-SNP: Strengthening VM Isolation with Integrity Protection and More}. 2020. \url{https://www.amd.com/system/files/TechDocs/SEV-SNP-strengthening-vm-isolation-with-integrity-protection-and-more.pdf} + +\bibitem{nvidia-cc} NVIDIA. \textit{Confidential Computing on NVIDIA H100 Tensor Core GPU}. 2024. \url{https://developer.nvidia.com/confidential-computing} + +\bibitem{azure-cvm} Microsoft. \textit{Azure Confidential Virtual Machines}. 2024. \url{https://learn.microsoft.com/azure/confidential-computing/confidential-vm-overview} + +\bibitem{litellm} BerriAI. \textit{LiteLLM: Call all LLM APIs using the OpenAI format}. 2024. \url{https://github.com/BerriAI/litellm} + +\bibitem{deepseek} DeepSeek AI. \textit{DeepSeek-V3 Technical Report}. 2024. \url{https://github.com/deepseek-ai/DeepSeek-V3} + +\bibitem{side-channels} Van Bulck, J., et al. \textit{Foreshadow: Extracting the Keys to the Intel SGX Kingdom}. USENIX Security 2018. + +\bibitem{slalom} Tramer, F. and Boneh, D. \textit{Slalom: Fast, Verifiable and Private Execution of Neural Networks in Trusted Hardware}. ICLR 2019. + +\bibitem{privado} Kumar, N., et al. \textit{Privado: Practical and Secure DNN Inference with TEEs}. arXiv:2023. + +\bibitem{dp-llm} Yu, D., et al. \textit{Differentially Private Fine-tuning of Language Models}. ICLR 2022. + +\bibitem{mpc-llm} Knott, B., et al. \textit{CrypTen: Secure Multi-Party Computation Meets Machine Learning}. NeurIPS 2021. + +\bibitem{he-llm} Chen, H., et al. \textit{Homomorphic Encryption for Machine Learning}. ACM Computing Surveys 2022. + +\end{thebibliography} + +\end{document} diff --git a/docs/tee-llm-infrastructure.tex b/docs/tee-llm-infrastructure.tex index 53ca1b3..0a801b4 100644 --- a/docs/tee-llm-infrastructure.tex +++ b/docs/tee-llm-infrastructure.tex @@ -31,12 +31,10 @@ urlcolor=blue } -\title{TrustedGenAi: Privacy-Preserving LLM Inference with\\Hardware-Attested Trusted Execution Environments} +\title{Privacy-Preserving LLM Inference with\\Hardware-Attested Trusted Execution Environments} \author[1]{Dzianis Vashchuk} -\author[2]{Claude-Opus-4.5\thanks{AI research assistant. Contributions include infrastructure implementation, documentation, and code generation.}} -\affil[1]{VibeTechnologies, \texttt{dzianis\_v@pm.me}} -\affil[2]{Anthropic} +\affil[1]{Vibe Technologies, LLC, \texttt{dzianisvv@gmail.com}} \date{January 2026} @@ -45,7 +43,7 @@ \maketitle \begin{abstract} -We present TrustedGenAi, an open-source infrastructure for deploying Large Language Model (LLM) inference within Trusted Execution Environments (TEEs) with cryptographic remote attestation. Our implementation runs self-hosted DeepSeek models on Azure Confidential VMs with Intel TDX, providing hardware-enforced memory encryption and verifiable privacy guarantees. We introduce a remote attestation API that enables clients to cryptographically verify TEE execution before submitting sensitive prompts. Our production deployment demonstrates practical feasibility with 12 tokens/second on CPU TEE and projects 150+ tokens/second on GPU TEE with NVIDIA H100 Confidential Computing. The complete infrastructure, including Terraform configurations and attestation services, is available at \url{https://github.com/VibeTechnologies/TrustedGenAi}. +We present an open-source infrastructure for deploying Large Language Model (LLM) inference within Trusted Execution Environments (TEEs) with cryptographic remote attestation. Our implementation runs self-hosted DeepSeek models on Azure Confidential VMs with Intel TDX, providing hardware-enforced memory encryption and verifiable privacy guarantees. We introduce a remote attestation API that enables clients to cryptographically verify TEE execution before submitting sensitive prompts. Our production deployment demonstrates practical feasibility with 12 tokens/second on CPU TEE and projects 150+ tokens/second on GPU TEE with NVIDIA H100 Confidential Computing. The complete infrastructure, including Terraform configurations and attestation services, is available at \url{https://github.com/VibeTechnologies/TrustedGenAi}. \end{abstract} \section{Introduction} @@ -92,13 +90,34 @@ \subsubsection{AMD Secure Encrypted Virtualization (SEV-SNP)} AMD SEV-SNP \cite{sev} provides similar guarantees with: \begin{itemize} - \item Per-VM encryption keys managed by AMD Secure Processor + \item Per-VM encryption keys managed by AMD Secure Processor (AES-128) \item Secure Nested Paging preventing hypervisor memory remapping attacks \item No guest modifications required (transparent to applications) \end{itemize} Azure offers AMD SEV-SNP via the DCasv5 series and NCCads\_H100\_v5 for GPU workloads. +\subsubsection{Platform Comparison} + +\begin{table}[H] +\centering +\caption{Intel TDX vs AMD SEV-SNP Comparison} +\begin{tabular}{@{}lll@{}} +\toprule +\textbf{Feature} & \textbf{Intel TDX} & \textbf{AMD SEV-SNP} \\ +\midrule +Memory Encryption & AES-256-XTS & AES-128 \\ +Key Management & CPU-managed & AMD Secure Processor \\ +Attestation Chain & Intel $\rightarrow$ Microsoft & AMD $\rightarrow$ Microsoft \\ +Azure VM Series & DCesv5 & DCasv5 \\ +Cost (4 vCPU, 16GB) & \$216/month & \$140/month \\ +GPU TEE Support & Google Cloud only & Azure (H100) \\ +\bottomrule +\end{tabular} +\end{table} + +Both platforms provide equivalent security for LLM inference: hardware-encrypted memory isolation from cloud operators. AMD SEV-SNP offers approximately 35\% cost savings on Azure, making it the recommended choice for cost-sensitive deployments. + \subsection{Remote Attestation} Remote attestation enables a client to cryptographically verify that code is running within a genuine TEE. The attestation flow consists of: @@ -268,6 +287,23 @@ \subsection{Infrastructure as Code} \subsection{CPU TEE Deployment} +Azure offers two CPU TEE options with equivalent security guarantees but different pricing: + +\begin{table}[H] +\centering +\caption{CPU TEE Platform Comparison (Azure, 4 vCPU / 16 GB)} +\begin{tabular}{@{}llll@{}} +\toprule +\textbf{Platform} & \textbf{VM Series} & \textbf{Cost/Month} & \textbf{Savings} \\ +\midrule +AMD SEV-SNP & DCasv5 (Standard\_DC4as\_v5) & \$140 & \textbf{35\% cheaper} \\ +Intel TDX & DCesv5 (Standard\_DC4es\_v5) & \$216 & baseline \\ +\bottomrule +\end{tabular} +\end{table} + +\textbf{Recommendation}: For equivalent security guarantees, AMD SEV-SNP provides 35\% cost savings. Both platforms offer hardware-encrypted memory, hypervisor isolation, and Microsoft-signed attestation. The choice between vendors is primarily economic. + Our production CPU TEE deployment uses: \begin{table}[H] @@ -288,9 +324,90 @@ \subsection{CPU TEE Deployment} \end{tabular} \end{table} -\subsection{GPU TEE Configuration (Projected)} +\subsection{GPU TEE with NVIDIA Confidential Computing} + +For production workloads requiring high throughput, GPU-accelerated TEE provides the optimal solution. NVIDIA H100 Tensor Core GPUs support Confidential Computing mode, extending the TEE boundary from CPU to GPU with hardware-based memory encryption \cite{nvidia-cc}. + +\subsubsection{NVIDIA H100 Confidential Computing Architecture} + +The NVIDIA H100 GPU implements Confidential Computing through: + +\begin{itemize} + \item \textbf{GPU Memory Encryption}: HBM3 memory is encrypted with keys managed by the GPU's security processor + \item \textbf{Secure Channel}: Encrypted PCIe communication between CPU TEE and GPU TEE + \item \textbf{GPU Attestation}: Hardware-rooted attestation reports verifying GPU confidential mode + \item \textbf{Isolation}: GPU memory isolated from host OS, hypervisor, and other VMs +\end{itemize} + +\subsubsection{Cloud Provider Availability} + +\begin{table}[H] +\centering +\caption{GPU TEE Availability by Cloud Provider} +\begin{tabular}{@{}llll@{}} +\toprule +\textbf{Provider} & \textbf{VM Series} & \textbf{GPU} & \textbf{TEE Type} \\ +\midrule +Azure & NCCads\_H100\_v5 & 1-4x H100 NVL & AMD SEV-SNP + NVIDIA CC \\ +Google Cloud & A3 Confidential & 8x H100 & Intel TDX + NVIDIA CC \\ +\bottomrule +\end{tabular} +\end{table} + +\subsubsection{Open-Weight Models on GPU TEE} -For production workloads requiring higher throughput, we provide Terraform configurations for NVIDIA H100 Confidential Computing. \textbf{Note: GPU TEE has not been deployed; the following specifications and performance numbers are projections based on hardware capabilities.} +A key advantage of TEE infrastructure is the ability to run any open-weight model with full privacy guarantees. The following table shows major open-weight model families compatible with GPU TEE deployment: + +\begin{table}[H] +\centering +\caption{Open-Weight Models Compatible with GPU TEE (NVIDIA H100)} +\begin{tabular}{@{}lllll@{}} +\toprule +\textbf{Model Family} & \textbf{Organization} & \textbf{Sizes} & \textbf{VRAM (FP16)} & \textbf{License} \\ +\midrule +DeepSeek-R1 & DeepSeek & 1.5B--671B & 3--80 GB & MIT \\ +DeepSeek-V3 & DeepSeek & 671B MoE & 80 GB (37B active) & MIT \\ +Llama 3.3 & Meta & 70B & 140 GB & Llama 3 \\ +Llama 3.1 & Meta & 8B--405B & 16--810 GB & Llama 3 \\ +Qwen 2.5 & Alibaba & 0.5B--72B & 1--144 GB & Apache 2.0 \\ +QwQ & Alibaba & 32B & 64 GB & Apache 2.0 \\ +Mistral Large 2 & Mistral & 123B & 246 GB & Apache 2.0 \\ +Mixtral 8x22B & Mistral & 141B MoE & 88 GB (44B active) & Apache 2.0 \\ +Codestral & Mistral & 22B & 44 GB & MNPL \\ +Kimi K2 & Moonshot & 1T MoE & 80 GB (32B active) & MIT \\ +MiniMax-M1 & MiniMax & 456B MoE & 80 GB (45B active) & Apache 2.0 \\ +Gemma 2 & Google & 2B--27B & 4--54 GB & Gemma \\ +Command R+ & Cohere & 104B & 208 GB & CC-BY-NC \\ +DBRX & Databricks & 132B MoE & 80 GB (36B active) & Apache 2.0 \\ +Phi-4 & Microsoft & 14B & 28 GB & MIT \\ +\bottomrule +\end{tabular} +\end{table} + +\textbf{Note}: MoE (Mixture of Experts) models show total parameters with active parameters in parentheses. Active parameters determine actual VRAM usage during inference. + +\subsubsection{DeepSeek Deployment on GPU TEE} + +DeepSeek models can be efficiently deployed on GPU TEE: + +\begin{table}[H] +\centering +\caption{DeepSeek Model Fit on NVIDIA H100 (94GB HBM3)} +\begin{tabular}{@{}llll@{}} +\toprule +\textbf{Model} & \textbf{Precision} & \textbf{VRAM} & \textbf{Fits on 1x H100} \\ +\midrule +DeepSeek-R1-Distill-1.5B & FP16 & 3 GB & Yes \\ +DeepSeek-R1-Distill-7B & FP16 & 14 GB & Yes \\ +DeepSeek-R1-Distill-14B & FP16 & 28 GB & Yes \\ +DeepSeek-R1-Distill-32B & FP16 & 64 GB & Yes \\ +DeepSeek-R1-Distill-70B & FP8 & 70 GB & Yes \\ +DeepSeek-V3 (671B MoE) & FP8 & 80-90 GB & Yes (37B active) \\ +\bottomrule +\end{tabular} +\end{table} + +\textbf{Note}: GPU TEE has not been deployed in our production environment; the following specifications are projections based on hardware capabilities and vendor documentation. \begin{table}[H] \centering @@ -304,12 +421,31 @@ \subsection{GPU TEE Configuration (Projected)} Memory & 320 GB \\ GPU & 1x NVIDIA H100 NVL (94 GB HBM3) \\ TEE Type & AMD SEV-SNP + NVIDIA CC \\ -Model & DeepSeek-R1-Distill-7B \\ +Model & DeepSeek-R1-Distill-70B (FP8) \\ +Projected Throughput & 150-300 tokens/sec \\ Cost & \$6,300/month \\ \bottomrule \end{tabular} \end{table} +\subsection{TPU TEE: Current Limitations} + +Google Cloud TPUs (Tensor Processing Units) currently \textbf{do not support Confidential Computing}. Unlike NVIDIA GPUs with hardware-level encryption, TPUs lack: + +\begin{itemize} + \item Hardware memory encryption for TPU HBM + \item Secure channel establishment with CPU TEE + \item Hardware-rooted attestation for TPU workloads +\end{itemize} + +\textbf{Implication}: For privacy-critical LLM inference requiring hardware attestation, NVIDIA H100 Confidential Computing is the only available GPU TEE option. TPU workloads cannot currently provide cryptographic privacy guarantees equivalent to CPU/GPU TEE deployments. + +\textbf{Future Outlook}: As confidential computing adoption grows, TPU TEE support may emerge. Organizations requiring TPU performance for large-scale LLM inference must currently choose between: +\begin{enumerate} + \item Performance (TPU without TEE) with policy-based privacy guarantees + \item Privacy (GPU TEE with H100) with hardware-verified guarantees +\end{enumerate} + \section{Evaluation} \subsection{Performance Benchmarks} @@ -410,24 +546,58 @@ \subsection{Future Work} \bibitem{nvidia-cc} NVIDIA. \textit{Confidential Computing on NVIDIA H100 Tensor Core GPU}. 2024. \url{https://developer.nvidia.com/confidential-computing} +\bibitem{nvidia-h100-arch} NVIDIA. \textit{NVIDIA H100 Tensor Core GPU Architecture Whitepaper}. 2022. \url{https://resources.nvidia.com/en-us-hopper-architecture/nvidia-h100-tensor-c} + +\bibitem{nvidia-cc-whitepaper} NVIDIA. \textit{NVIDIA Confidential Computing: Protecting Data in Use on GPUs}. 2023. \url{https://www.nvidia.com/en-us/data-center/solutions/confidential-computing/} + \bibitem{azure-cvm} Microsoft. \textit{Azure Confidential Virtual Machines}. 2024. \url{https://learn.microsoft.com/azure/confidential-computing/confidential-vm-overview} +\bibitem{azure-attestation} Microsoft. \textit{Azure Attestation Overview}. 2024. \url{https://learn.microsoft.com/azure/attestation/overview} + +\bibitem{azure-acc-h100} Microsoft. \textit{NCCadsH100v5-series Confidential VMs}. 2024. \url{https://learn.microsoft.com/azure/virtual-machines/nccadsh100-v5-series} + +\bibitem{gcp-cvm} Google Cloud. \textit{Confidential VM Overview}. 2024. \url{https://cloud.google.com/confidential-computing/confidential-vm/docs/about-cvm} + +\bibitem{gcp-gpu-cvm} Google Cloud. \textit{Create a Confidential VM Instance with GPU}. 2024. \url{https://cloud.google.com/confidential-computing/confidential-vm/docs/create-a-confidential-vm-instance-with-gpu} + \bibitem{litellm} BerriAI. \textit{LiteLLM: Call all LLM APIs using the OpenAI format}. 2024. \url{https://github.com/BerriAI/litellm} \bibitem{deepseek} DeepSeek AI. \textit{DeepSeek-V3 Technical Report}. 2024. \url{https://github.com/deepseek-ai/DeepSeek-V3} +\bibitem{deepseek-r1} DeepSeek AI. \textit{DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning}. 2025. \url{https://arxiv.org/abs/2501.12948} + +\bibitem{ollama} Ollama. \textit{Ollama: Run Large Language Models Locally}. 2024. \url{https://github.com/ollama/ollama} + +\bibitem{vllm} Kwon, W., et al. \textit{Efficient Memory Management for Large Language Model Serving with PagedAttention}. SOSP 2023. \url{https://arxiv.org/abs/2309.06180} + \bibitem{side-channels} Van Bulck, J., et al. \textit{Foreshadow: Extracting the Keys to the Intel SGX Kingdom}. USENIX Security 2018. +\bibitem{sgx-attacks} Costan, V. and Devadas, S. \textit{Intel SGX Explained}. IACR Cryptology ePrint Archive 2016. \url{https://eprint.iacr.org/2016/086} + \bibitem{slalom} Tramer, F. and Boneh, D. \textit{Slalom: Fast, Verifiable and Private Execution of Neural Networks in Trusted Hardware}. ICLR 2019. \bibitem{privado} Kumar, N., et al. \textit{Privado: Practical and Secure DNN Inference with TEEs}. arXiv:2023. +\bibitem{mlcapsule} Hanzlik, L., et al. \textit{MLCapsule: Guarded Offline Deployment of Machine Learning as a Service}. CVPR 2021. + \bibitem{dp-llm} Yu, D., et al. \textit{Differentially Private Fine-tuning of Language Models}. ICLR 2022. \bibitem{mpc-llm} Knott, B., et al. \textit{CrypTen: Secure Multi-Party Computation Meets Machine Learning}. NeurIPS 2021. \bibitem{he-llm} Chen, H., et al. \textit{Homomorphic Encryption for Machine Learning}. ACM Computing Surveys 2022. +\bibitem{secureml} Mohassel, P. and Zhang, Y. \textit{SecureML: A System for Scalable Privacy-Preserving Machine Learning}. IEEE S\&P 2017. + +\bibitem{marlin} Marlin Protocol. \textit{Oyster: Verifiable Serverless Computing}. 2024. \url{https://www.marlin.org/oyster} + +\bibitem{phala} Phala Network. \textit{Phala: Trustless Cloud Computing on TEE}. 2024. \url{https://phala.network/} + +\bibitem{oasis} Oasis Labs. \textit{Oasis Network: Privacy-Preserving Smart Contracts}. 2024. \url{https://oasisprotocol.org/} + +\bibitem{terraform} HashiCorp. \textit{Terraform: Infrastructure as Code}. 2024. \url{https://www.terraform.io/} + +\bibitem{cloudflare-tunnel} Cloudflare. \textit{Cloudflare Tunnel: Secure Connections Without Public IPs}. 2024. \url{https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/} + \end{thebibliography} \end{document} diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..2a69fbd --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,42 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/azure/azapi" { + version = "1.15.0" + constraints = "~> 1.0" + hashes = [ + "h1:pO/phGY+TxMEKQ+ffYj+vUIvG5A1tno/sZYDb/yyA/w=", + "zh:0627a8bc77254debc25dc0c7b62e055138217c97b03221e593c3c56dc7550671", + "zh:2fe045f07070ef75d0bec4b0595a74c14394daa838ddb964e2fd23cc98c40c34", + "zh:343009f39c957883b2c06145a5954e524c70f93585f943f1ea3d28ef6995d0d0", + "zh:53fe9ab54485aaebc9b91e27a10bce2729a1c95b1399079e631dc6bb9e3f27dc", + "zh:63c407e7dc04d178d4798c17ad489d9cc92f7d1941d7f4a3f560b95908b6107b", + "zh:7d6fc2b432b264f036bb80ab2b2ba67f80a5d98da8a8c322aa097833dad598c9", + "zh:7ec49c0a8799d469eb6e2a1f856693f9862f1b73f5ed70adc1b346e5a4c6458d", + "zh:889704f10319d301d677539d788fc82a7c73608ab78cb93e1280ac2be39e6e00", + "zh:90b4b07405b7cde9ebae3b034cb5bb5dd18484d1b95bd250f905451f1e86ac3f", + "zh:92aa9c241a8cb2a6d81ad47bc007c119f8b818464a960ebaf39008766c361e6b", + "zh:f28fbd0a2c59e239b53067bc1adc691be444876bcb2d4f78d310f549724da6e0", + "zh:ffb15e0ddfa505d0e9b75341570199076ae574887124f398162b1ead9376b25f", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.117.1" + constraints = "~> 3.0" + hashes = [ + "h1:j6wnjpHfBcQC4xd3ZYquaIPIIR46xJQs7rxwPdSOZos=", + "zh:0c513676836e3c50d004ece7d2624a8aff6faac14b833b96feeac2e4bc2c1c12", + "zh:50ea01ada95bae2f187db9e926e463f45d860767a85ebc59160414e00e76c35d", + "zh:52c2a9edacc06b3f72153f5ef6daca0761c6292158815961fe37f60bc576a3d7", + "zh:618eed2a06b19b1a025b45b05891846d570a6a1cca4d23f4942f5a99e1f747ae", + "zh:61cde5d3165d7e5ec311d5d89486819cd605c1b2d54611b5c97bd4e97dba2762", + "zh:6a873358d5031fc222f5e05f029d1237f3dce8345c767665f393283dfa2627f6", + "zh:afdd80064b2a04da311856feb4ed45f77ff4df6c356e8c2b10afb51fe7e61c70", + "zh:b09113df7e0e8c8959539bd22bae6c39faeb269ba3c4cd948e742f5cf58c35fb", + "zh:d340db7973109761cfc27d52aa02560363337c908b2c99b3628adc5a70a99d5b", + "zh:d5a577226ebc8c65e8f19384878a86acc4b51ede4b4a82d37c3b331b0efcd4a7", + "zh:e2962b147f9e71732df8dbc74940c10d20906f3c003cbfaa1eb9fabbf601a9f0", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/cpu-tee-amd-sev.tf b/terraform/cpu-tee-amd-sev.tf new file mode 100644 index 0000000..6449cdf --- /dev/null +++ b/terraform/cpu-tee-amd-sev.tf @@ -0,0 +1,426 @@ +# ============================================================================== +# CPU TEE - AMD SEV-SNP (DCasv5 Series) +# ============================================================================== +# +# AMD Secure Encrypted Virtualization - Secure Nested Paging (SEV-SNP) +# - Memory encryption: AES-128 with AMD Secure Processor managed keys +# - Isolation: Hardware-enforced from hypervisor, other VMs, host OS +# - Attestation: AMD + Microsoft signed +# +# Cost: ~$140/month (Standard_DC4as_v5) - 35% CHEAPER than Intel TDX +# Performance: ~12 tokens/sec with DeepSeek-R1 1.5B +# +# Enable: terraform apply -var="enable_amd_sev=true" +# Verify: ssh azureuser@ "dmesg | grep -i sev" +# +# ============================================================================== + +variable "enable_amd_sev" { + description = "Enable AMD SEV-SNP CPU TEE deployment" + type = bool + default = false +} + +variable "amd_sev_vm_size" { + description = "AMD SEV-SNP VM size (DCasv5 series)" + type = string + default = "Standard_DC4as_v5" # 4 vCPU, 16GB RAM, ~$0.19/hr (~$140/mo) +} + +variable "amd_sev_model" { + description = "Model to run on AMD SEV-SNP TEE" + type = string + default = "deepseek-r1:1.5b" +} + +# ============================================================================== +# Network Infrastructure +# ============================================================================== + +resource "azurerm_virtual_network" "amd_sev" { + count = var.enable_amd_sev ? 1 : 0 + name = "tee-amd-sev-vnet" + location = var.location + resource_group_name = azurerm_resource_group.main.name + address_space = ["10.20.0.0/16"] + + tags = { + tee_type = "AMD-SEV-SNP" + } +} + +resource "azurerm_subnet" "amd_sev" { + count = var.enable_amd_sev ? 1 : 0 + name = "tee-subnet" + resource_group_name = azurerm_resource_group.main.name + virtual_network_name = azurerm_virtual_network.amd_sev[0].name + address_prefixes = ["10.20.0.0/24"] +} + +resource "azurerm_network_security_group" "amd_sev" { + count = var.enable_amd_sev ? 1 : 0 + name = "tee-amd-sev-nsg" + location = var.location + resource_group_name = azurerm_resource_group.main.name + + security_rule { + name = "SSH" + priority = 1001 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "*" + destination_address_prefix = "*" + } + + security_rule { + name = "LiteLLM-API" + priority = 1002 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "4000" + source_address_prefix = "*" + destination_address_prefix = "*" + } + + security_rule { + name = "Attestation-API" + priority = 1003 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "4001" + source_address_prefix = "*" + destination_address_prefix = "*" + } + + tags = { + tee_type = "AMD-SEV-SNP" + } +} + +resource "azurerm_subnet_network_security_group_association" "amd_sev" { + count = var.enable_amd_sev ? 1 : 0 + subnet_id = azurerm_subnet.amd_sev[0].id + network_security_group_id = azurerm_network_security_group.amd_sev[0].id +} + +resource "azurerm_public_ip" "amd_sev" { + count = var.enable_amd_sev ? 1 : 0 + name = "tee-amd-sev-pip" + location = var.location + resource_group_name = azurerm_resource_group.main.name + allocation_method = "Static" + sku = "Standard" + + tags = { + tee_type = "AMD-SEV-SNP" + } +} + +resource "azurerm_network_interface" "amd_sev" { + count = var.enable_amd_sev ? 1 : 0 + name = "tee-amd-sev-nic" + location = var.location + resource_group_name = azurerm_resource_group.main.name + + ip_configuration { + name = "internal" + subnet_id = azurerm_subnet.amd_sev[0].id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = azurerm_public_ip.amd_sev[0].id + } + + tags = { + tee_type = "AMD-SEV-SNP" + } +} + +# ============================================================================== +# Cloud-init for AMD SEV-SNP VM +# ============================================================================== + +locals { + amd_sev_cloud_init = base64encode(<<-EOF + #!/bin/bash + set -ex + + export HOME=/root + exec > /var/log/tee-init.log 2>&1 + + echo "=== AMD SEV-SNP TEE Setup ===" + echo "Platform: AMD SEV-SNP" + echo "VM Size: ${var.amd_sev_vm_size}" + echo "Model: ${var.amd_sev_model}" + date + + # Verify AMD SEV-SNP is active + echo "Verifying AMD SEV-SNP..." + dmesg | grep -i "SEV-SNP" && echo "AMD SEV-SNP VERIFIED" || echo "WARNING: SEV-SNP not detected" + dmesg | grep -i "Memory Encryption" || true + + # Install ollama + echo "Installing ollama..." + curl -fsSL https://ollama.com/install.sh | sh + systemctl enable ollama + systemctl start ollama + sleep 10 + + # Pull model + echo "Pulling model: ${var.amd_sev_model}" + HOME=/root ollama pull ${var.amd_sev_model} + + # Configure ollama to listen on all interfaces + mkdir -p /etc/systemd/system/ollama.service.d + cat > /etc/systemd/system/ollama.service.d/override.conf <<'OVERRIDE' + [Service] + Environment="OLLAMA_HOST=0.0.0.0" + OVERRIDE + systemctl daemon-reload + systemctl restart ollama + + # Install LiteLLM + echo "Installing LiteLLM..." + apt-get update -qq + apt-get install -y python3-pip python3-venv -qq + python3 -m venv /opt/litellm + /opt/litellm/bin/pip install -q litellm[proxy] + + # LiteLLM config + cat > /opt/litellm/config.yaml <<'CONFIG' + model_list: + - model_name: deepseek-r1 + litellm_params: + model: ollama/${var.amd_sev_model} + api_base: http://localhost:11434 + - model_name: deepseek-r1-1.5b + litellm_params: + model: ollama/deepseek-r1:1.5b + api_base: http://localhost:11434 + + general_settings: + master_key: ${var.tee_api_key} + CONFIG + + # LiteLLM systemd service + cat > /etc/systemd/system/litellm.service <<'SERVICE' + [Unit] + Description=LiteLLM Proxy + After=network.target ollama.service + + [Service] + Type=simple + ExecStart=/opt/litellm/bin/litellm --config /opt/litellm/config.yaml --port 4000 --host 0.0.0.0 + Restart=always + RestartSec=10 + + [Install] + WantedBy=multi-user.target + SERVICE + + systemctl daemon-reload + systemctl enable litellm + systemctl start litellm + + # Attestation API service + cat > /opt/attestation-api.py <<'ATTESTATION' + #!/usr/bin/env python3 + import json + import subprocess + import http.server + import urllib.request + + class AttestationHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == '/attestation': + self.send_response(200) + self.send_header('Content-type', 'application/json') + self.send_header('Access-Control-Allow-Origin', '*') + self.end_headers() + + # Get TEE info from dmesg + dmesg = subprocess.run(['dmesg'], capture_output=True, text=True) + tee_lines = [l for l in dmesg.stdout.split('\n') + if 'SEV' in l or 'Memory Encryption' in l] + + # Get Azure attestation document (PKCS7 signed by Microsoft) + try: + req = urllib.request.Request( + 'http://169.254.169.254/metadata/attested/document?api-version=2021-02-01', + headers={'Metadata': 'true'} + ) + with urllib.request.urlopen(req, timeout=5) as resp: + azure_attestation = json.loads(resp.read()) + except Exception as e: + azure_attestation = {"error": str(e)} + + # Get TPM PCR values + try: + pcr = subprocess.run(['tpm2_pcrread', 'sha256'], capture_output=True, text=True) + tpm_pcr = pcr.stdout + except: + tpm_pcr = "TPM not available" + + response = { + "platform": "AMD-SEV-SNP", + "vm_size": "${var.amd_sev_vm_size}", + "tee_verified": any('SEV' in l for l in tee_lines), + "azure_attestation": azure_attestation, + "tpm_pcr_sha256": tpm_pcr, + "tee_dmesg": tee_lines[:5] + } + self.wfile.write(json.dumps(response, indent=2).encode()) + else: + self.send_response(404) + self.end_headers() + + def log_message(self, format, *args): + pass + + if __name__ == '__main__': + server = http.server.HTTPServer(('0.0.0.0', 4001), AttestationHandler) + print('Attestation API running on port 4001') + server.serve_forever() + ATTESTATION + + chmod +x /opt/attestation-api.py + + cat > /etc/systemd/system/attestation.service <<'SERVICE' + [Unit] + Description=TEE Attestation API + After=network.target + + [Service] + Type=simple + ExecStart=/usr/bin/python3 /opt/attestation-api.py + Restart=always + RestartSec=10 + + [Install] + WantedBy=multi-user.target + SERVICE + + systemctl daemon-reload + systemctl enable attestation + systemctl start attestation + + echo "=== AMD SEV-SNP Setup Complete ===" + date + EOF + ) +} + +# ============================================================================== +# AMD SEV-SNP Confidential VM +# ============================================================================== + +resource "azapi_resource" "amd_sev_vm" { + count = var.enable_amd_sev ? 1 : 0 + type = "Microsoft.Compute/virtualMachines@2024-03-01" + name = "tee-amd-sev" + location = var.location + parent_id = azurerm_resource_group.main.id + + body = jsonencode({ + properties = { + hardwareProfile = { + vmSize = var.amd_sev_vm_size + } + securityProfile = { + securityType = "ConfidentialVM" + uefiSettings = { + secureBootEnabled = true + vTpmEnabled = true + } + } + storageProfile = { + imageReference = { + publisher = "Canonical" + offer = "0001-com-ubuntu-confidential-vm-jammy" + sku = "22_04-lts-cvm" + version = "latest" + } + osDisk = { + createOption = "FromImage" + managedDisk = { + storageAccountType = "Premium_LRS" + securityProfile = { + securityEncryptionType = "VMGuestStateOnly" + } + } + diskSizeGB = 128 + } + } + osProfile = { + computerName = "tee-amd-sev" + adminUsername = "azureuser" + customData = local.amd_sev_cloud_init + linuxConfiguration = { + disablePasswordAuthentication = true + ssh = { + publicKeys = [ + { + path = "/home/azureuser/.ssh/authorized_keys" + keyData = file(var.ssh_public_key_path) + } + ] + } + } + } + networkProfile = { + networkInterfaces = [ + { + id = azurerm_network_interface.amd_sev[0].id + properties = { + primary = true + } + } + ] + } + } + zones = ["1"] + }) + + tags = { + tee_type = "AMD-SEV-SNP" + tee_enabled = "true" + model = var.amd_sev_model + cost = "$140/month" + } + + depends_on = [azurerm_network_interface.amd_sev] +} + +# ============================================================================== +# Outputs +# ============================================================================== + +output "amd_sev_enabled" { + description = "Whether AMD SEV-SNP TEE is enabled" + value = var.enable_amd_sev +} + +output "amd_sev_public_ip" { + description = "Public IP of AMD SEV-SNP VM" + value = var.enable_amd_sev ? azurerm_public_ip.amd_sev[0].ip_address : null +} + +output "amd_sev_ssh" { + description = "SSH command for AMD SEV-SNP VM" + value = var.enable_amd_sev ? "ssh azureuser@${azurerm_public_ip.amd_sev[0].ip_address}" : null +} + +output "amd_sev_api" { + description = "LiteLLM API endpoint" + value = var.enable_amd_sev ? "http://${azurerm_public_ip.amd_sev[0].ip_address}:4000/v1" : null +} + +output "amd_sev_attestation" { + description = "Attestation API endpoint" + value = var.enable_amd_sev ? "http://${azurerm_public_ip.amd_sev[0].ip_address}:4001/attestation" : null +} diff --git a/terraform/cpu-tee-intel-tdx.tf b/terraform/cpu-tee-intel-tdx.tf new file mode 100644 index 0000000..a06b2c7 --- /dev/null +++ b/terraform/cpu-tee-intel-tdx.tf @@ -0,0 +1,424 @@ +# ============================================================================== +# CPU TEE - Intel TDX (DCesv5 Series) +# ============================================================================== +# +# Intel Trust Domain Extensions (TDX) on Azure DCesv5 series +# - Memory encryption: AES-256-XTS with CPU-managed keys +# - Isolation: Hardware-enforced from hypervisor, other VMs, host OS +# - Attestation: Intel + Microsoft signed +# +# Cost: ~$216/month (Standard_DC4es_v5) +# Performance: ~12 tokens/sec with DeepSeek-R1 1.5B +# +# Enable: terraform apply -var="enable_intel_tdx=true" +# Verify: ssh azureuser@ "dmesg | grep -i tdx" +# +# ============================================================================== + +variable "enable_intel_tdx" { + description = "Enable Intel TDX CPU TEE deployment" + type = bool + default = false +} + +variable "intel_tdx_vm_size" { + description = "Intel TDX VM size (DCesv5 series)" + type = string + default = "Standard_DC4es_v5" # 4 vCPU, 16GB RAM, ~$0.30/hr (~$216/mo) +} + +variable "intel_tdx_model" { + description = "Model to run on Intel TDX TEE" + type = string + default = "deepseek-r1:1.5b" +} + +# ============================================================================== +# Network Infrastructure +# ============================================================================== + +resource "azurerm_virtual_network" "intel_tdx" { + count = var.enable_intel_tdx ? 1 : 0 + name = "tee-intel-tdx-vnet" + location = var.location + resource_group_name = azurerm_resource_group.main.name + address_space = ["10.10.0.0/16"] + + tags = { + tee_type = "Intel-TDX" + } +} + +resource "azurerm_subnet" "intel_tdx" { + count = var.enable_intel_tdx ? 1 : 0 + name = "tee-subnet" + resource_group_name = azurerm_resource_group.main.name + virtual_network_name = azurerm_virtual_network.intel_tdx[0].name + address_prefixes = ["10.10.0.0/24"] +} + +resource "azurerm_network_security_group" "intel_tdx" { + count = var.enable_intel_tdx ? 1 : 0 + name = "tee-intel-tdx-nsg" + location = var.location + resource_group_name = azurerm_resource_group.main.name + + security_rule { + name = "SSH" + priority = 1001 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "*" + destination_address_prefix = "*" + } + + security_rule { + name = "LiteLLM-API" + priority = 1002 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "4000" + source_address_prefix = "*" + destination_address_prefix = "*" + } + + security_rule { + name = "Attestation-API" + priority = 1003 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "4001" + source_address_prefix = "*" + destination_address_prefix = "*" + } + + tags = { + tee_type = "Intel-TDX" + } +} + +resource "azurerm_subnet_network_security_group_association" "intel_tdx" { + count = var.enable_intel_tdx ? 1 : 0 + subnet_id = azurerm_subnet.intel_tdx[0].id + network_security_group_id = azurerm_network_security_group.intel_tdx[0].id +} + +resource "azurerm_public_ip" "intel_tdx" { + count = var.enable_intel_tdx ? 1 : 0 + name = "tee-intel-tdx-pip" + location = var.location + resource_group_name = azurerm_resource_group.main.name + allocation_method = "Static" + sku = "Standard" + + tags = { + tee_type = "Intel-TDX" + } +} + +resource "azurerm_network_interface" "intel_tdx" { + count = var.enable_intel_tdx ? 1 : 0 + name = "tee-intel-tdx-nic" + location = var.location + resource_group_name = azurerm_resource_group.main.name + + ip_configuration { + name = "internal" + subnet_id = azurerm_subnet.intel_tdx[0].id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = azurerm_public_ip.intel_tdx[0].id + } + + tags = { + tee_type = "Intel-TDX" + } +} + +# ============================================================================== +# Cloud-init for Intel TDX VM +# ============================================================================== + +locals { + intel_tdx_cloud_init = base64encode(<<-EOF + #!/bin/bash + set -ex + + export HOME=/root + exec > /var/log/tee-init.log 2>&1 + + echo "=== Intel TDX TEE Setup ===" + echo "Platform: Intel TDX" + echo "VM Size: ${var.intel_tdx_vm_size}" + echo "Model: ${var.intel_tdx_model}" + date + + # Verify Intel TDX is active + echo "Verifying Intel TDX..." + dmesg | grep -i "Intel TDX" && echo "Intel TDX VERIFIED" || echo "WARNING: TDX not detected" + + # Install ollama + echo "Installing ollama..." + curl -fsSL https://ollama.com/install.sh | sh + systemctl enable ollama + systemctl start ollama + sleep 10 + + # Pull model + echo "Pulling model: ${var.intel_tdx_model}" + HOME=/root ollama pull ${var.intel_tdx_model} + + # Configure ollama to listen on all interfaces + mkdir -p /etc/systemd/system/ollama.service.d + cat > /etc/systemd/system/ollama.service.d/override.conf <<'OVERRIDE' + [Service] + Environment="OLLAMA_HOST=0.0.0.0" + OVERRIDE + systemctl daemon-reload + systemctl restart ollama + + # Install LiteLLM + echo "Installing LiteLLM..." + apt-get update -qq + apt-get install -y python3-pip python3-venv -qq + python3 -m venv /opt/litellm + /opt/litellm/bin/pip install -q litellm[proxy] + + # LiteLLM config + cat > /opt/litellm/config.yaml <<'CONFIG' + model_list: + - model_name: deepseek-r1 + litellm_params: + model: ollama/${var.intel_tdx_model} + api_base: http://localhost:11434 + - model_name: deepseek-r1-1.5b + litellm_params: + model: ollama/deepseek-r1:1.5b + api_base: http://localhost:11434 + + general_settings: + master_key: ${var.tee_api_key} + CONFIG + + # LiteLLM systemd service + cat > /etc/systemd/system/litellm.service <<'SERVICE' + [Unit] + Description=LiteLLM Proxy + After=network.target ollama.service + + [Service] + Type=simple + ExecStart=/opt/litellm/bin/litellm --config /opt/litellm/config.yaml --port 4000 --host 0.0.0.0 + Restart=always + RestartSec=10 + + [Install] + WantedBy=multi-user.target + SERVICE + + systemctl daemon-reload + systemctl enable litellm + systemctl start litellm + + # Attestation API service + cat > /opt/attestation-api.py <<'ATTESTATION' + #!/usr/bin/env python3 + import json + import subprocess + import http.server + import urllib.request + + class AttestationHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == '/attestation': + self.send_response(200) + self.send_header('Content-type', 'application/json') + self.send_header('Access-Control-Allow-Origin', '*') + self.end_headers() + + # Get TEE info + dmesg = subprocess.run(['dmesg'], capture_output=True, text=True) + tee_lines = [l for l in dmesg.stdout.split('\n') if 'TDX' in l or 'Memory Encryption' in l] + + # Get Azure attestation + try: + req = urllib.request.Request( + 'http://169.254.169.254/metadata/attested/document?api-version=2021-02-01', + headers={'Metadata': 'true'} + ) + with urllib.request.urlopen(req, timeout=5) as resp: + azure_attestation = json.loads(resp.read()) + except: + azure_attestation = None + + # Get TPM PCR values + try: + pcr = subprocess.run(['tpm2_pcrread', 'sha256'], capture_output=True, text=True) + tpm_pcr = pcr.stdout + except: + tpm_pcr = "TPM not available" + + response = { + "platform": "Intel-TDX", + "vm_size": "${var.intel_tdx_vm_size}", + "tee_verified": len(tee_lines) > 0, + "azure_attestation": azure_attestation, + "tpm_pcr_sha256": tpm_pcr, + "tee_dmesg": tee_lines[:5] + } + self.wfile.write(json.dumps(response, indent=2).encode()) + else: + self.send_response(404) + self.end_headers() + + def log_message(self, format, *args): + pass + + if __name__ == '__main__': + server = http.server.HTTPServer(('0.0.0.0', 4001), AttestationHandler) + print('Attestation API running on port 4001') + server.serve_forever() + ATTESTATION + + chmod +x /opt/attestation-api.py + + cat > /etc/systemd/system/attestation.service <<'SERVICE' + [Unit] + Description=TEE Attestation API + After=network.target + + [Service] + Type=simple + ExecStart=/usr/bin/python3 /opt/attestation-api.py + Restart=always + RestartSec=10 + + [Install] + WantedBy=multi-user.target + SERVICE + + systemctl daemon-reload + systemctl enable attestation + systemctl start attestation + + echo "=== Intel TDX Setup Complete ===" + date + EOF + ) +} + +# ============================================================================== +# Intel TDX Confidential VM +# ============================================================================== + +resource "azapi_resource" "intel_tdx_vm" { + count = var.enable_intel_tdx ? 1 : 0 + type = "Microsoft.Compute/virtualMachines@2024-03-01" + name = "tee-intel-tdx" + location = var.location + parent_id = azurerm_resource_group.main.id + + body = jsonencode({ + properties = { + hardwareProfile = { + vmSize = var.intel_tdx_vm_size + } + securityProfile = { + securityType = "ConfidentialVM" + uefiSettings = { + secureBootEnabled = true + vTpmEnabled = true + } + } + storageProfile = { + imageReference = { + publisher = "Canonical" + offer = "0001-com-ubuntu-confidential-vm-jammy" + sku = "22_04-lts-cvm" + version = "latest" + } + osDisk = { + createOption = "FromImage" + managedDisk = { + storageAccountType = "Premium_LRS" + securityProfile = { + securityEncryptionType = "VMGuestStateOnly" + } + } + diskSizeGB = 128 + } + } + osProfile = { + computerName = "tee-intel-tdx" + adminUsername = "azureuser" + customData = local.intel_tdx_cloud_init + linuxConfiguration = { + disablePasswordAuthentication = true + ssh = { + publicKeys = [ + { + path = "/home/azureuser/.ssh/authorized_keys" + keyData = file(var.ssh_public_key_path) + } + ] + } + } + } + networkProfile = { + networkInterfaces = [ + { + id = azurerm_network_interface.intel_tdx[0].id + properties = { + primary = true + } + } + ] + } + } + zones = ["1"] + }) + + tags = { + tee_type = "Intel-TDX" + tee_enabled = "true" + model = var.intel_tdx_model + cost = "$216/month" + } + + depends_on = [azurerm_network_interface.intel_tdx] +} + +# ============================================================================== +# Outputs +# ============================================================================== + +output "intel_tdx_enabled" { + description = "Whether Intel TDX TEE is enabled" + value = var.enable_intel_tdx +} + +output "intel_tdx_public_ip" { + description = "Public IP of Intel TDX VM" + value = var.enable_intel_tdx ? azurerm_public_ip.intel_tdx[0].ip_address : null +} + +output "intel_tdx_ssh" { + description = "SSH command for Intel TDX VM" + value = var.enable_intel_tdx ? "ssh azureuser@${azurerm_public_ip.intel_tdx[0].ip_address}" : null +} + +output "intel_tdx_api" { + description = "LiteLLM API endpoint" + value = var.enable_intel_tdx ? "http://${azurerm_public_ip.intel_tdx[0].ip_address}:4000/v1" : null +} + +output "intel_tdx_attestation" { + description = "Attestation API endpoint" + value = var.enable_intel_tdx ? "http://${azurerm_public_ip.intel_tdx[0].ip_address}:4001/attestation" : null +}