diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef43d72f4..40222f04c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,10 +33,11 @@ jobs: - run: pnpm install --frozen-lockfile - run: pnpm build - run: node scripts/test/simulator-packages.mjs - - name: Verify simulator infrastructure profiles + - name: Verify infrastructure profiles run: | pnpm nx run @moltzap/simulator:local-profile-check pnpm nx run @moltzap/simulator:gke-profile-check + pnpm nx run @moltzap/evals:phoenix-terraform-check - run: pnpm typecheck - run: pnpm lint # Exact runs fail when a required project target disappears; run-many diff --git a/packages/evals/README.md b/packages/evals/README.md index 31a0c7681..3798d5322 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -208,5 +208,11 @@ Set `PHOENIX_API_KEY` when required. Repeated publication reconciles the stable case dataset, one experiment per condition, and every report attempt before returning the Phoenix experiment URLs. +`phoenix/` provisions a hosted Phoenix for that endpoint: a Cloud Run service on +Cloud SQL Postgres, authenticated from its first revision, whose lifecycle is +independent of the GKE cluster. `PHOENIX_HOST` comes from its `service_url` +output; `PHOENIX_API_KEY` is created in the application against a named account. +See `phoenix/README.md`. + This repository has static coverage for both profiles. It does not claim that a live local or GKE evaluation has completed successfully. diff --git a/packages/evals/package.json b/packages/evals/package.json index 2b101d762..b612b15c7 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -16,6 +16,7 @@ "resume": "nx run @moltzap/evals:resume", "calibrate": "nx run @moltzap/evals:calibrate", "publish": "nx run @moltzap/evals:publish", + "phoenix:terraform:check": "nx run @moltzap/evals:phoenix-terraform-check", "typecheck:tests": "nx run @moltzap/evals:typecheck:tests" }, "dependencies": { diff --git a/packages/evals/phoenix/README.md b/packages/evals/phoenix/README.md new file mode 100644 index 000000000..c974b74bc --- /dev/null +++ b/packages/evals/phoenix/README.md @@ -0,0 +1,243 @@ +# Hosted Phoenix + +Terraform for the Arize Phoenix instance that the evaluation publisher writes +to. It is a Cloud Run service on a Cloud SQL Postgres database, authenticated +from its first revision, in the same project as everything else this repository +provisions. + +This is the destination of `pnpm nx run @moltzap/evals:publish`. Nothing else in +the repository depends on it, and the publisher reaches it over plain HTTPS, so +the module owns no TypeScript and the package owns no Terraform beyond it. + +## Why Cloud Run, and why not under the GKE profile + +Both answers are the same answer. `packages/simulator/gke` provisions a cluster +whose lifecycle is deliberately short: `cluster.sh down` parks it and +`cluster.sh delete` destroys it, and both are routine — that profile's README +costs deletion at about eight minutes. Published dashboards are the opposite. +They are the retained result of a run, read long after the cluster that produced +them is gone. A dashboard that disappears whenever the thing it describes is +torn down is not a dashboard. + +So Phoenix does not run on the cluster, and its Terraform does not live beside +the cluster's, where one `terraform destroy` could take both. Cloud Run is the +smallest thing that survives: no cluster to keep alive between runs, and the +state that matters is in Cloud SQL rather than in the compute at all, so the +service can be deleted and re-applied without touching the data. + +It sits in `packages/evals` because that is where the contract is consumed — +`PHOENIX_HOST` and `PHOENIX_API_KEY` are read by `packages/evals/src/phoenix.ts` +and by nothing in the simulator. The layout mirrors the GKE profile's shape, a +named directory holding a README and a `terraform/` root, one package over. + +## Why not Phoenix Cloud + +Two reasons, and the second is decisive. + +Evaluation data would leave the project. The whole arrangement here — a +dedicated project, a private database, secrets that never leave Secret Manager — +exists so that run evidence stays inside a boundary that can be reasoned about. +Managed hosting moves it outside for a convenience this module does not need. + +More concretely, the publisher cannot authenticate to it. `phoenix.ts` builds +its client with `Authorization: Bearer `. Phoenix Cloud expects +its own `api_key` header style. Pointing the existing publisher at managed +hosting would require changing the client, and this workstream deliberately +touches no TypeScript. + +## What the publisher needs + +Two environment variables, both derived from a single apply: + +```bash +PHOENIX_HOST="$(terraform -chdir=packages/evals/phoenix/terraform output -raw service_url)" +PHOENIX_API_KEY=… # created in the Phoenix UI, see below +``` + +`PHOENIX_HOST` is the Cloud Run origin. `PHOENIX_API_KEY` is a Phoenix API key +and is deliberately not a Terraform output: keys are created in the application, +against a named account, and can be revoked there without an apply. + +`PHOENIX_HOST` means two different things depending on which side of the wire +reads it. To the publisher it is the origin to call. To the Phoenix server it is +the address to bind, which is why the service does not set it — the container +keeps the default `0.0.0.0` and sets only `PHOENIX_PORT`. Do not copy the +publisher's value into the service. + +## The account model + +Authentication is on in the first revision. `PHOENIX_ENABLE_AUTH` and +`PHOENIX_SECRET` are set before the service ever answers a request, so there is +no window during which an unauthenticated Phoenix is reachable, and no later +migration from open to closed that someone can forget to perform. + +Phoenix bootstraps one account, `admin@localhost`, whose initial password is +normally the literal string `admin`. This module overrides it with a generated +password held in Secret Manager, so the well-known default is never valid. The +`admin_bootstrap` output prints the login URL and the exact `gcloud` command +that reads it; take it from there rather than from a name written down here, +which goes stale the moment `name_prefix` changes. + +That password is a bootstrap credential, not a shared one. Sign in with it, +change it immediately, and then create a **named account per person** from the +admin settings page. Everyone reading dashboards gets their own login and their +own API keys. A password passed around a team cannot be revoked for one person, +does not say who published an experiment, and survives their departure. + +The strong password policy is enabled, so the accounts created that way must +carry a real password rather than a memorable one. + +`PHOENIX_ADMIN_SECRET` is not set, and should not be. It is a standing bearer +token that authenticates as the first system user, which would hand every holder +of one string full administrative access with no name attached to it and no way +to revoke one holder. Named accounts with their own keys cost one page of +clicking and are revocable individually. + +## Applying + +```bash +cp packages/evals/phoenix/terraform/terraform.tfvars.example \ + packages/evals/phoenix/terraform/terraform.tfvars +# set project_id + +terraform -chdir=packages/evals/phoenix/terraform init +terraform -chdir=packages/evals/phoenix/terraform apply +``` + +The first apply enables five APIs, so it may need to be run twice on a project +where they were not already on — Google reports a service as enabled before it +is uniformly usable, and the Cloud Run service agent that pulls images is +created as a side effect of enabling `run.googleapis.com`. + +First start is slow. Phoenix runs its full migration set against an empty +database before the port answers, which is why the startup probe allows five +minutes. + +Then take ownership: + +```bash +terraform -chdir=packages/evals/phoenix/terraform output admin_bootstrap +``` + +## Destroying + +The Cloud SQL instance and the Cloud Run service both refuse to be destroyed +while `deletion_protection` is on, which is the default here — the point of the +module is that this data outlives the cluster, so the default is the opposite of +the GKE profile's. Removing it is a deliberate, separate step: + +```bash +terraform -chdir=packages/evals/phoenix/terraform apply -var deletion_protection=false +terraform -chdir=packages/evals/phoenix/terraform destroy +``` + +Cloud SQL reserves a deleted instance's name for roughly a week, so an immediate +re-apply under the same `name_prefix` will fail. Take a final export first if the +dashboards matter; `destroy` does not. + +## How it is put together + +**Image.** `phoenix_image` is a Docker Hub repository path pinned by digest; +two `validation` blocks reject a tag and reject a fully qualified reference. The +variable's own description carries the rationale and the command that resolves a +new digest. It is mirrored through an Artifact Registry remote repository, which +preserves the digest, so the reference still names exactly one image. + +**Database.** Cloud Run mounts the Cloud SQL Auth proxy socket at +`/cloudsql/INSTANCE_CONNECTION_NAME` through a `cloud_sql_instance` volume — the +`google_cloud_run_v2_service` spelling of the `run.googleapis.com/cloudsql-instances` +annotation, same connector, same socket. The instance declares no authorized +network and never should: reaching it is an IAM decision, granted by +`roles/cloudsql.client` on the runtime service account, and the proxy encrypts +and authenticates independently of any address allowlist. + +The connection string is the one detail here worth reading closely: + +``` +postgresql://phoenix:PASSWORD@/phoenix?host=/cloudsql/PROJECT:REGION:INSTANCE +``` + +The authority is empty and the socket directory is a query parameter. Phoenix +builds its engine with SQLAlchemy, which preserves query parameters and hands +`host` to asyncpg, and asyncpg reads a leading slash as a Unix socket directory. +Putting the path in the host slot instead does not work, because the instance +connection name contains colons that are then read as a port — which is also why +the `PHOENIX_POSTGRES_HOST` family cannot be used here. That family interpolates +its host into the authority directly, so it can address a TCP endpoint and +nothing else. + +This was verified against the pinned image before it was written down: Phoenix +run against a Postgres reachable only through a socket directory named +`/cloudsql/my-proj:us-central1:phoenix-test` completed its migrations and +created its full schema over that socket. + +**Secrets.** The database password, the JWT signing secret, and the initial +admin password are generated by `random_password` and stored in Secret Manager, +and the service reads all three through `secret_key_ref` rather than as +plaintext environment variables. Access is granted per secret, so the runtime +identity can read these three and nothing the project acquires later. + +The database password is not stored a second time on its own. It exists inside +`moltzap-phoenix-database-url`, and giving one credential two homes means a +rotation that updates one and not the other. Read it out of the URL when a human +needs it. + +The generated values are in Terraform state in plaintext. That is inherent to +generating them in Terraform, and passing them in as variables instead would put +them in state too. What follows from it is that the state is itself a secret: +keep it in a remote backend with restricted access rather than on a laptop. The +alternative — creating the secrets out of band and having Terraform reference +them by name — keeps state clean at the cost of a manual step before the first +apply, and is a reasonable trade to make later. + +**Hardening.** The image's agent assistant is disabled. It can execute shell +commands and reach the network from inside the container, is on by default, and +nothing here uses it; leaving it enabled on an internet-reachable service would +be an unforced error. Usage telemetry is off, which is consistent with the +reason for self-hosting at all. Session cookies require TLS, which Cloud Run +serves exclusively. + +`PHOENIX_CSRF_TRUSTED_ORIGINS` is not set, because the service URL does not +exist until the service does. If the browser UI reports a CSRF failure on +sign-in, set it to the `service_url` output in a follow-up apply. + +**Ingress.** `allUsers` holds `roles/run.invoker` — a requirement, not a +shortcut, for the reason set out on the `allow_public_invoker` variable: Cloud +Run's IAM authentication and the publisher's bearer key cannot both own the +`Authorization` header. Phoenix's own authentication is therefore the access +control. A project under a domain-restricted-sharing org policy will reject the +`allUsers` grant and needs an exception, or a different design. + +## What it costs + +This module provisions standing infrastructure, which the GKE profile +deliberately does not. `min_instances = 1` with always-allocated CPU keeps one +Cloud Run instance billed continuously, and Cloud SQL bills whether or not +anyone opens a dashboard; together they are the dominant line, well above the +storage the data itself uses. Confirm current rates against the pricing pages +before committing to it. + +That is the price of the property this module exists for — a dashboard that is +up when someone follows a link to it, holding spans that were buffered when the +last publish finished. `min_instances = 0` removes most of the compute charge +and is the right setting if the instance is only ever read through the UI: the +cost is a slow first request, since a cold start replays the migration check, +and a narrow window in which a publish's final spans can be lost. Nothing else +in the module assumes one is running. + +## Checking it + +```bash +pnpm nx run @moltzap/evals:phoenix-terraform-check +``` + +`terraform fmt -check`, then `init -backend=false`, then `validate`. CI runs it +beside the simulator profile checks. `terraform-check.sh` documents what the +check does and does not prove. + +## Upstream contracts + +- [Phoenix self-hosting configuration](https://arize.com/docs/phoenix/self-hosting/configuration) +- [Connect to Cloud SQL for PostgreSQL from Cloud Run](https://cloud.google.com/sql/docs/postgres/connect-run) +- [Deploy container images to Cloud Run](https://cloud.google.com/run/docs/deploying) +- [Artifact Registry remote repositories](https://cloud.google.com/artifact-registry/docs/repositories/remote-overview) diff --git a/packages/evals/phoenix/terraform-check.sh b/packages/evals/phoenix/terraform-check.sh new file mode 100755 index 000000000..254167f81 --- /dev/null +++ b/packages/evals/phoenix/terraform-check.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Static validation for the hosted Phoenix module. +# +# This never contacts Google Cloud. It selects no project, reads no +# credentials, and touches no state: `init -backend=false` resolves providers +# against the committed lock file and stops there. It therefore proves the +# configuration is well formed and internally consistent, and proves nothing +# about whether an apply would succeed against a real project. +# +# Resolving providers does reach the Terraform registry the first time it runs +# in a working tree. That is a genuine failure when it fails, not a skip: a +# check that quietly passes without validating anything is worse than one that +# is absent. +set -euo pipefail + +module_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/terraform" && pwd)" + +if ! command -v terraform >/dev/null 2>&1; then + echo "phoenix-terraform-check: skipped, terraform is not installed" >&2 + exit 0 +fi + +terraform -chdir="$module_root" fmt -check -recursive +terraform -chdir="$module_root" init -backend=false -input=false -no-color >/dev/null +terraform -chdir="$module_root" validate -no-color diff --git a/packages/evals/phoenix/terraform/.gitignore b/packages/evals/phoenix/terraform/.gitignore new file mode 100644 index 000000000..1ec89e723 --- /dev/null +++ b/packages/evals/phoenix/terraform/.gitignore @@ -0,0 +1,5 @@ +.terraform/ +*.tfplan +*.tfstate +*.tfstate.* +terraform.tfvars diff --git a/packages/evals/phoenix/terraform/.terraform.lock.hcl b/packages/evals/phoenix/terraform/.terraform.lock.hcl new file mode 100644 index 000000000..55fccadd2 --- /dev/null +++ b/packages/evals/phoenix/terraform/.terraform.lock.hcl @@ -0,0 +1,48 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.42.0" + constraints = "7.42.0" + hashes = [ + "h1:6qNk18qjViinYxnjAEix5O+qHPMmXXHdzCU1IpEJLqg=", + "h1:JqhNUoY3Jw6g4lfznOd4B8qXh2lNetk5/W+dWVZJUwI=", + "h1:OgWsoxTL8UjiDXmmqrK3twhvUEFI0IfIr3NzCmUeAGk=", + "h1:gN0gSVFKLscRyG28ngPvWKEp8JnWa3iASU+mdG+H7Wo=", + "zh:30b25728203b9208a167fac3f9880c10242fc5accdd29ba01b21355566fc4e3d", + "zh:4468f6ea772e991d890724e44f628a24dae44c9028af654469454d05b00b10ec", + "zh:4dfa4f7bcd72ea89f6f3f7411d88bf9a1f060699830e1f1f85bf32102be754b3", + "zh:59cf73879f10ad9d29ff8ad96559a476e70695bed26b84b6189728129674618c", + "zh:73a7966ae1c6db8a3dc31eb43f05dddd27a47f3ff42e25f62594fc0d5b438412", + "zh:7c2ea415fb06147cf9834b2169d75a52bd979ad291bed32c94d9a9307316f7ba", + "zh:962efdd3dee2b98860528555b0616ca0c8987dfc6c5e5df6d1c025c9b22c2f26", + "zh:c4a5ca9f20cbfbdcb88064d53d16f2ce8038e1ddc3303c7261152856728700b5", + "zh:ca56a9477177530737d07feea70bb99309414a7c18aa62f975644138606e1faf", + "zh:d0c6db8b1da363f69087569716ed96a3a6dadb4b872f598d442d867bd0706fa9", + "zh:ddd2472052e0c5c3fab7cffae8376e8855c35ad8614ba8631f7e448e72a41f21", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.7.2" + constraints = "3.7.2" + hashes = [ + "h1:356j/3XnXEKr9nyicLUufzoF4Yr6hRy481KIxRVpK0c=", + "h1:KG4NuIBl1mRWU0KD/BGfCi1YN/j3F7H4YgeeM7iSdNs=", + "h1:Lmv2TxyKKm9Vt4uxcPZHw1uf0Ax/yYizJlilbLSZN8E=", + "h1:hkKSY5xI4R1H4Yrg10HHbtOoxZif2dXa9HFPSbaVg5o=", + "zh:14829603a32e4bc4d05062f059e545a91e27ff033756b48afbae6b3c835f508f", + "zh:1527fb07d9fea400d70e9e6eb4a2b918d5060d604749b6f1c361518e7da546dc", + "zh:1e86bcd7ebec85ba336b423ba1db046aeaa3c0e5f921039b3f1a6fc2f978feab", + "zh:24536dec8bde66753f4b4030b8f3ef43c196d69cccbea1c382d01b222478c7a3", + "zh:29f1786486759fad9b0ce4fdfbbfece9343ad47cd50119045075e05afe49d212", + "zh:4d701e978c2dd8604ba1ce962b047607701e65c078cb22e97171513e9e57491f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b8434212eef0f8c83f5a90c6d76feaf850f6502b61b53c329e85b3b281cba34", + "zh:ac8a23c212258b7976e1621275e3af7099e7e4a3d4478cf8d5d2a27f3bc3e967", + "zh:b516ca74431f3df4c6cf90ddcdb4042c626e026317a33c53f0b445a3d93b720d", + "zh:dc76e4326aec2490c1600d6871a95e78f9050f9ce427c71707ea412a2f2f1a62", + "zh:eac7b63e86c749c7d48f527671c7aee5b4e26c10be6ad7232d6860167f99dbb0", + ] +} diff --git a/packages/evals/phoenix/terraform/main.tf b/packages/evals/phoenix/terraform/main.tf new file mode 100644 index 000000000..884069d07 --- /dev/null +++ b/packages/evals/phoenix/terraform/main.tf @@ -0,0 +1,430 @@ +locals { + required_services = toset([ + "artifactregistry.googleapis.com", + "iam.googleapis.com", + "run.googleapis.com", + "secretmanager.googleapis.com", + "sqladmin.googleapis.com", + ]) + + # Phoenix's own default, kept rather than moved to 8080, so that the port in + # the container matches every Phoenix document and log line. + container_port = 6006 + + # Cloud Run mounts the Cloud SQL Auth proxy socket under this directory. The + # instance connection name contains colons; nothing here may quote or escape + # them, because the proxy publishes the directory verbatim. + cloudsql_mount_path = "/cloudsql" + + # Cloud Run pulls images as its own service agent, not as the runtime + # identity below, so the repository grant has to name this principal. It is + # spelled out rather than looked up because the resource that reads service + # agents lives only in the beta provider. + run_service_agent = "service-${data.google_project.current.number}@serverless-robot-prod.iam.gserviceaccount.com" + + phoenix_image_reference = join("/", [ + "${var.region}-docker.pkg.dev", + var.project_id, + google_artifact_registry_repository.upstream.repository_id, + var.phoenix_image, + ]) + + # Phoenix builds its engine with SQLAlchemy, which preserves query parameters + # and hands `host` to asyncpg, and asyncpg reads a host beginning with a + # slash as a Unix socket directory. The empty authority is what keeps the + # socket path out of the host slot, where the colons in the instance + # connection name would be read as a port. + database_url = format( + "postgresql://%s:%s@/%s?host=%s/%s", + google_sql_user.phoenix.name, + random_password.database.result, + google_sql_database.phoenix.name, + local.cloudsql_mount_path, + google_sql_database_instance.phoenix.connection_name, + ) + + # Every entry is generated here and read only by the service account below. + # Driving the secret, its version, and its IAM grant from one map is what + # keeps a secret from ever existing without the grant that makes it readable. + secrets = { + "database-url" = local.database_url + "phoenix-secret" = random_password.phoenix_secret.result + "admin-initial-password" = random_password.admin_initial.result + } +} + +data "google_project" "current" { + project_id = var.project_id +} + +resource "google_project_service" "required" { + for_each = local.required_services + + project = var.project_id + service = each.value + disable_on_destroy = false +} + +# Google recommends pulling third-party images through a remote repository +# rather than from Docker Hub directly, and the digest survives the mirror +# unchanged, so pinning still names exactly one image. +resource "google_artifact_registry_repository" "upstream" { + project = var.project_id + location = var.region + repository_id = "${var.name_prefix}-docker-hub-remote" + description = "Pull-through mirror of Docker Hub for the pinned Phoenix image" + format = "DOCKER" + mode = "REMOTE_REPOSITORY" + + remote_repository_config { + description = "Docker Hub" + + docker_repository { + public_repository = "DOCKER_HUB" + } + } + + depends_on = [google_project_service.required] +} + +resource "google_artifact_registry_repository_iam_member" "run_image_reader" { + project = var.project_id + location = google_artifact_registry_repository.upstream.location + repository = google_artifact_registry_repository.upstream.name + role = "roles/artifactregistry.reader" + member = "serviceAccount:${local.run_service_agent}" +} + +resource "google_service_account" "phoenix" { + project = var.project_id + account_id = var.name_prefix + display_name = "MoltZap hosted Phoenix runtime" + + depends_on = [google_project_service.required] +} + +# Reaching the database is an IAM decision, not a network one. The instance +# accepts no unauthenticated path at all; this grant is what lets the Cloud Run +# proxy open the socket. +resource "google_project_iam_member" "cloudsql_client" { + project = var.project_id + role = "roles/cloudsql.client" + member = "serviceAccount:${google_service_account.phoenix.email}" +} + +# Passwords are generated here rather than supplied so that no operator ever +# holds one, and so that a first apply cannot produce a revision with a +# placeholder credential. `special = false` on the two values that end up +# inside a URL removes any question of percent-encoding. +resource "random_password" "database" { + length = 40 + special = false + min_lower = 4 + min_upper = 4 + min_numeric = 4 +} + +# Phoenix rejects a signing secret shorter than 32 characters or missing a +# digit or a lowercase letter, and refuses to start rather than downgrading. +resource "random_password" "phoenix_secret" { + length = 48 + special = false + min_lower = 4 + min_upper = 4 + min_numeric = 4 +} + +# Satisfies the strong password policy enabled on the service below: at least +# twelve characters with an uppercase letter, a lowercase letter, a digit, and +# a symbol. The symbol set excludes quoting and shell metacharacters so an +# operator can paste it into a login form without escaping anything. +resource "random_password" "admin_initial" { + length = 24 + min_lower = 4 + min_upper = 4 + min_numeric = 4 + min_special = 2 + override_special = "!#%*-_=+" +} + +resource "google_sql_database_instance" "phoenix" { + project = var.project_id + name = var.name_prefix + region = var.region + database_version = var.sql_database_version + deletion_protection = var.deletion_protection + + settings { + tier = var.sql_tier + edition = "ENTERPRISE" + availability_type = "ZONAL" + disk_type = "PD_SSD" + disk_size = 10 + disk_autoresize = true + deletion_protection_enabled = var.deletion_protection + + backup_configuration { + enabled = true + start_time = "07:00" + point_in_time_recovery_enabled = var.point_in_time_recovery + + backup_retention_settings { + retained_backups = var.backup_retention_count + retention_unit = "COUNT" + } + } + + # No authorized network is declared and none should be. Every connection + # arrives through the Cloud SQL Auth proxy, which authenticates with IAM + # and encrypts independently of any address allowlist. + ip_configuration { + ipv4_enabled = true + ssl_mode = "ENCRYPTED_ONLY" + } + + maintenance_window { + day = 7 + hour = 8 + update_track = "stable" + } + } + + lifecycle { + # Autoresize grows the disk without Terraform's involvement. Left as drift, + # the next plan would try to put it back, and Cloud SQL refuses to shrink a + # disk — so the apply would fail rather than the disk shrinking. disk_size + # is a floor, and this is what keeps it one. + ignore_changes = [settings[0].disk_size] + } + + depends_on = [google_project_service.required] +} + +resource "google_sql_database" "phoenix" { + project = var.project_id + name = "phoenix" + instance = google_sql_database_instance.phoenix.name +} + +resource "google_sql_user" "phoenix" { + project = var.project_id + name = "phoenix" + instance = google_sql_database_instance.phoenix.name + password = random_password.database.result +} + +resource "google_secret_manager_secret" "phoenix" { + for_each = local.secrets + + project = var.project_id + secret_id = "${var.name_prefix}-${each.key}" + + replication { + auto {} + } + + depends_on = [google_project_service.required] +} + +resource "google_secret_manager_secret_version" "phoenix" { + for_each = local.secrets + + secret = google_secret_manager_secret.phoenix[each.key].id + secret_data = each.value +} + +# Iterating the secrets themselves, rather than a second list of names, is what +# makes it impossible to add a secret and forget its grant. Access is granted +# per secret rather than project-wide, so the runtime identity can read these +# three and nothing else the project later acquires. +resource "google_secret_manager_secret_iam_member" "phoenix_accessor" { + for_each = google_secret_manager_secret.phoenix + + project = var.project_id + secret_id = each.value.secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.phoenix.email}" +} + +resource "google_cloud_run_v2_service" "phoenix" { + project = var.project_id + name = var.name_prefix + location = var.region + ingress = "INGRESS_TRAFFIC_ALL" + deletion_protection = var.deletion_protection + + template { + service_account = google_service_account.phoenix.email + execution_environment = "EXECUTION_ENVIRONMENT_GEN2" + timeout = "600s" + + # One instance, not a variable. Phoenix is not deployed here as a + # horizontally scaled service: several instances behind one URL would share + # a database without sharing their in-memory span queues. + scaling { + min_instance_count = var.min_instances + max_instance_count = 1 + } + + volumes { + name = "cloudsql" + + cloud_sql_instance { + instances = [google_sql_database_instance.phoenix.connection_name] + } + } + + containers { + image = local.phoenix_image_reference + + ports { + container_port = local.container_port + } + + resources { + limits = { + cpu = var.cpu + memory = var.memory + } + + # Phoenix drains its span queue on background tasks. Throttling the CPU + # between requests, which is the Cloud Run default, would stall that + # drain until the next request happened to arrive. + cpu_idle = false + startup_cpu_boost = true + } + + volume_mounts { + name = "cloudsql" + mount_path = local.cloudsql_mount_path + } + + env { + name = "PHOENIX_PORT" + value = tostring(local.container_port) + } + + # Postgres is the store of record. The working directory only ever holds + # scratch, so it points at the writable in-memory filesystem rather than + # at a home directory that may not exist in the image. + env { + name = "PHOENIX_WORKING_DIR" + value = "/tmp/phoenix" + } + + # Authentication is on in the first revision, not added later, so there + # is no window in which an unauthenticated Phoenix is reachable. + env { + name = "PHOENIX_ENABLE_AUTH" + value = "True" + } + + env { + name = "PHOENIX_ENABLE_STRONG_PASSWORD_POLICY" + value = "True" + } + + # Cloud Run serves TLS only, so the session cookies can require it. + env { + name = "PHOENIX_USE_SECURE_COOKIES" + value = "True" + } + + # Phoenix ships an assistant that can execute shell commands and reach + # the network from inside the container. Nothing here uses it, and this + # service is reachable from the internet by design, so it is off. + env { + name = "PHOENIX_DISABLE_AGENT_ASSISTANT" + value = "True" + } + + # Keeping evaluation data inside the project is the reason this is + # self-hosted; usage telemetry leaving it would undercut that. + env { + name = "PHOENIX_TELEMETRY_ENABLED" + value = "False" + } + + env { + name = "PHOENIX_SQL_DATABASE_URL" + + value_source { + secret_key_ref { + secret = google_secret_manager_secret.phoenix["database-url"].secret_id + version = "latest" + } + } + } + + env { + name = "PHOENIX_SECRET" + + value_source { + secret_key_ref { + secret = google_secret_manager_secret.phoenix["phoenix-secret"].secret_id + version = "latest" + } + } + } + + # Only consulted while the default admin record is being created. Once it + # exists, changing this has no effect and the password is changed in the + # application. + env { + name = "PHOENIX_DEFAULT_ADMIN_INITIAL_PASSWORD" + + value_source { + secret_key_ref { + secret = google_secret_manager_secret.phoenix["admin-initial-password"].secret_id + version = "latest" + } + } + } + + # Generous, because the first start of an empty database runs every + # Phoenix migration before the port answers. + startup_probe { + initial_delay_seconds = 10 + period_seconds = 10 + timeout_seconds = 5 + failure_threshold = 30 + + http_get { + path = "/healthz" + port = local.container_port + } + } + + liveness_probe { + period_seconds = 30 + timeout_seconds = 5 + failure_threshold = 3 + + http_get { + path = "/healthz" + port = local.container_port + } + } + } + } + + depends_on = [ + google_artifact_registry_repository_iam_member.run_image_reader, + google_project_iam_member.cloudsql_client, + google_secret_manager_secret_iam_member.phoenix_accessor, + google_secret_manager_secret_version.phoenix, + ] +} + +# Cloud Run stops at the network edge here and Phoenix authenticates every +# request itself. See the allow_public_invoker variable for why the two cannot +# both own the Authorization header. +resource "google_cloud_run_v2_service_iam_member" "public_invoker" { + count = var.allow_public_invoker ? 1 : 0 + + project = var.project_id + location = google_cloud_run_v2_service.phoenix.location + name = google_cloud_run_v2_service.phoenix.name + role = "roles/run.invoker" + member = "allUsers" +} diff --git a/packages/evals/phoenix/terraform/outputs.tf b/packages/evals/phoenix/terraform/outputs.tf new file mode 100644 index 000000000..5742dcfff --- /dev/null +++ b/packages/evals/phoenix/terraform/outputs.tf @@ -0,0 +1,39 @@ +output "service_url" { + description = "Phoenix origin. This is PHOENIX_HOST for the evaluation publisher and the address to open in a browser." + value = google_cloud_run_v2_service.phoenix.uri +} + +output "instance_connection_name" { + description = "Cloud SQL instance connection name, in project:region:instance form. Cloud Run publishes the Auth proxy socket under /cloudsql using exactly this string, and `gcloud sql connect` takes it too." + value = google_sql_database_instance.phoenix.connection_name +} + +output "service_account_email" { + description = "Runtime identity of the service. It holds Cloud SQL client on the project and secret access on this module's three secrets, and nothing else." + value = google_service_account.phoenix.email +} + +output "phoenix_image_reference" { + description = "Digest-pinned image the service actually runs, resolved through the Artifact Registry mirror." + value = local.phoenix_image_reference +} + +# The values are deliberately absent. Naming the secrets is enough to reach +# them with `gcloud secrets versions access`, which leaves an audit record; +# emitting them here would copy three credentials into every plan file and +# state snapshot that already holds them once. +output "secret_ids" { + description = "Secret Manager secret IDs holding the database URL, the JWT signing secret, and the initial admin password." + value = { + for id, secret in google_secret_manager_secret.phoenix : id => secret.secret_id + } +} + +output "admin_bootstrap" { + description = "How to take ownership of the instance on first deploy." + value = { + login_url = "${google_cloud_run_v2_service.phoenix.uri}/login" + email = "admin@localhost" + password_via = "gcloud secrets versions access latest --project ${var.project_id} --secret ${google_secret_manager_secret.phoenix["admin-initial-password"].secret_id}" + } +} diff --git a/packages/evals/phoenix/terraform/terraform.tfvars.example b/packages/evals/phoenix/terraform/terraform.tfvars.example new file mode 100644 index 000000000..a967f6983 --- /dev/null +++ b/packages/evals/phoenix/terraform/terraform.tfvars.example @@ -0,0 +1 @@ +project_id = "replace-with-project-id" diff --git a/packages/evals/phoenix/terraform/variables.tf b/packages/evals/phoenix/terraform/variables.tf new file mode 100644 index 000000000..466031270 --- /dev/null +++ b/packages/evals/phoenix/terraform/variables.tf @@ -0,0 +1,171 @@ +variable "project_id" { + description = "Google Cloud project hosting the Phoenix service, its database, and its secrets." + type = string + + validation { + condition = can(regex("^[a-z][a-z0-9-]{4,28}[a-z0-9]$", var.project_id)) + error_message = "project_id must be a valid Google Cloud project ID." + } +} + +variable "region" { + description = "Cloud Run region, Cloud SQL region, and Artifact Registry location. One region for all three keeps the database socket and the service in the same failure domain." + type = string + default = "us-central1" +} + +variable "name_prefix" { + description = <<-EOT + Prefix for every named object here: the Cloud Run service, the Cloud SQL + instance, the runtime service account, and the three Secret Manager secrets. + + Kept short because the service account ID derived from it is capped at + thirty characters by IAM. + EOT + type = string + default = "moltzap-phoenix" + + validation { + condition = can(regex("^[a-z][a-z0-9-]{2,19}[a-z0-9]$", var.name_prefix)) + error_message = "name_prefix must be 4-21 lowercase alphanumeric or hyphen characters starting with a letter." + } +} + +variable "phoenix_image" { + description = <<-EOT + Phoenix image as a Docker Hub repository path pinned by digest. + + This is deliberately the upstream path and not a fully qualified reference: + the module mirrors it through its own Artifact Registry remote repository, + so the registry host, project, and repository are derived rather than + restated. Only the repository path and digest are an input. + + The digest, not a tag, is the deployed identity. Resolve a new one with: + + docker manifest inspect arizephoenix/phoenix:version-X.Y.Z + + or, without a Docker daemon, read the `Docker-Content-Digest` header: + + TOKEN=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:arizephoenix/phoenix:pull" | jq -r .token) + curl -sI -H "Authorization: Bearer $TOKEN" \ + -H "Accept: application/vnd.oci.image.index.v1+json" \ + https://registry-1.docker.io/v2/arizephoenix/phoenix/manifests/version-X.Y.Z \ + | grep -i docker-content-digest + + The default is the multi-architecture index digest of + `arizephoenix/phoenix:version-19.18.0`, resolved 2026-08-06, which was also + `:latest` at that moment. Cloud Run selects the linux/amd64 child manifest. + EOT + type = string + default = "arizephoenix/phoenix@sha256:65211b52795a0f8b67e29ec1a0787d4b683acb3c5450d2c30a30e0ad670bd3a8" + + validation { + condition = can(regex("@sha256:[0-9a-f]{64}$", var.phoenix_image)) + error_message = "phoenix_image must be pinned by digest, ending in @sha256: followed by 64 lowercase hex characters. A mutable tag is not a valid input." + } + + validation { + condition = !can(regex("^[^/]*[.:][^/]*/", var.phoenix_image)) + error_message = "phoenix_image must be a Docker Hub repository path such as arizephoenix/phoenix@sha256:..., not a fully qualified registry reference. The registry host is derived from the module's Artifact Registry remote repository." + } +} + +variable "sql_tier" { + description = <<-EOT + Cloud SQL machine type. + + This database holds evaluation dashboards, not a serving workload: it is + written once per publish and read by a handful of people. The smallest + shared-core tier carries that comfortably. + EOT + type = string + default = "db-g1-small" +} + +variable "sql_database_version" { + description = "Cloud SQL Postgres major version. Phoenix runs its own migrations against whatever it finds, so this moves independently of the image." + type = string + default = "POSTGRES_16" + + validation { + condition = can(regex("^POSTGRES_[0-9]+$", var.sql_database_version)) + error_message = "sql_database_version must be a POSTGRES_ identifier." + } +} + +variable "point_in_time_recovery" { + description = "Retain write-ahead logs so the database can be restored to an arbitrary moment rather than only to a nightly backup. Costs log storage." + type = bool + default = true +} + +variable "backup_retention_count" { + description = "Number of automated backups retained." + type = number + default = 7 + + validation { + condition = var.backup_retention_count >= 1 && floor(var.backup_retention_count) == var.backup_retention_count + error_message = "backup_retention_count must be a positive integer." + } +} + +variable "deletion_protection" { + description = <<-EOT + Refuse to destroy the Cloud SQL instance and the Cloud Run service. + + Defaults to on, the opposite of the GKE profile, because the entire reason + this module exists is that the dashboards must survive events that delete + the cluster. Set it to false and apply once before `terraform destroy`. + EOT + type = bool + default = true +} + +variable "cpu" { + description = "Cloud Run CPU limit. At least one full CPU is required for an always-allocated instance, which this service needs so that queued spans flush between requests." + type = string + default = "1" +} + +variable "memory" { + description = "Cloud Run memory limit. Phoenix buffers spans in memory and the working directory is an in-memory filesystem, so this is not purely process headroom." + type = string + default = "2Gi" +} + +variable "min_instances" { + description = <<-EOT + Floor on running instances. + + One, not zero. Phoenix runs database migrations at startup and holds an + in-memory span queue, so a scale-to-zero service would pay a migration on + every cold start and could drop spans buffered when the last instance went + away. Zero is defensible if the instance is only ever read through the UI. + EOT + type = number + default = 1 + + validation { + condition = var.min_instances >= 0 && floor(var.min_instances) == var.min_instances + error_message = "min_instances must be a non-negative integer." + } +} + +variable "allow_public_invoker" { + description = <<-EOT + Grant `roles/run.invoker` to allUsers so that Cloud Run itself does not + demand a Google identity. + + This is required by the publisher, not a convenience: the Phoenix client + authenticates with `Authorization: Bearer `, and Cloud Run + IAM authentication claims that same header for its own identity token. The + two cannot both own it. Turning this off means every caller must present a + Google identity token instead, which the publisher cannot do. + + What stands between the internet and the data is Phoenix's authentication, + which this module enables in the first revision. + EOT + type = bool + default = true +} diff --git a/packages/evals/phoenix/terraform/versions.tf b/packages/evals/phoenix/terraform/versions.tf new file mode 100644 index 000000000..fa465816d --- /dev/null +++ b/packages/evals/phoenix/terraform/versions.tf @@ -0,0 +1,19 @@ +terraform { + required_version = ">= 1.8.0, < 2.0.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "= 7.42.0" + } + random = { + source = "hashicorp/random" + version = "= 3.7.2" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} diff --git a/packages/evals/project.json b/packages/evals/project.json index b6ec9d68e..0fddda4cd 100644 --- a/packages/evals/project.json +++ b/packages/evals/project.json @@ -85,6 +85,21 @@ "command": "tsc -p tsconfig.test.json" } }, + "phoenix-terraform-check": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "{projectRoot}/phoenix/**/*", + "{projectRoot}/phoenix/terraform/.terraform.lock.hcl", + { + "runtime": "terraform version 2>/dev/null || echo none" + } + ], + "options": { + "cwd": "packages/evals", + "command": "bash phoenix/terraform-check.sh" + } + }, "arch:check": { "executor": "nx:run-commands", "options": {