diff --git a/.config/nextest.toml b/.config/nextest.toml index afe4a8d1..47c36274 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -7,3 +7,9 @@ test-threads = "num-cpus" # Longer timeout for e2e tests slow-timeout = { period = "60s", terminate-after = 2 } + +# Override timeout for real e2e tests (5 minutes) +# Matches tests whose function names start with real_test_ (all tests in real_e2e_* files) +[[profile.default.overrides]] +filter = 'test(/^real_test_/)' +slow-timeout = { period = "500s", terminate-after = 1 } diff --git a/.github/workflows/test-real.yml b/.github/workflows/test-real.yml new file mode 100644 index 00000000..cb5157ff --- /dev/null +++ b/.github/workflows/test-real.yml @@ -0,0 +1,119 @@ +name: Real E2E Tests + +on: + pull_request: + types: [ opened, ready_for_review ] # When PR is ready for review (not draft) + issue_comment: + types: [ created ] # Only trigger on PR comments with @tester test + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + CARGO_INCREMENTAL: 0 # Faster CI builds + +jobs: + test_real: + name: Real E2E Test Suite + # Only run on: + # - push / pull_request events, or + # - issue_comment events where the comment is on a PR and mentions @claude + if: | + github.event_name != 'issue_comment' || + (github.event_name == 'issue_comment' && + github.event.issue.pull_request && + github.event.comment.body == 'test') + runs-on: ubuntu_large_x64 + environment: Cloud API test env + permissions: + contents: read + pull-requests: read + issues: read + + services: + postgres: + image: postgres:15 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: platform_api + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Get PR head SHA + id: pr-head + env: + GH_TOKEN: ${{ github.token }} + run: | + set -e + PR_NUMBER=${{ github.event.issue.number }} + echo "Fetching PR #${PR_NUMBER} head SHA..." + + if ! PR_HEAD_SHA=$(gh pr view "$PR_NUMBER" --repo ${{ github.repository }} --json headRefOid --jq '.headRefOid' 2>&1); then + echo "Error: Failed to fetch PR #${PR_NUMBER} information" >&2 + echo "$PR_HEAD_SHA" >&2 + exit 1 + fi + + if [ -z "$PR_HEAD_SHA" ]; then + echo "Error: PR head SHA is empty for PR #${PR_NUMBER}" >&2 + exit 1 + fi + + echo "sha=$PR_HEAD_SHA" >> $GITHUB_OUTPUT + echo "PR head SHA: $PR_HEAD_SHA" + + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ steps.pr-head.outputs.sha }} + + - name: Install Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + components: rustfmt, clippy + + - name: Cache Rust dependencies + uses: swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + # Run only real_e2e tests (tests with function names starting with real_test_) + - name: Run real E2E tests + run: cargo nextest run -p api --tests -E 'test(/^real_test_/)' + env: + POSTGRES_PRIMARY_APP_ID: ${{ secrets.POSTGRES_PRIMARY_APP_ID }} + DATABASE_HOST: localhost + DATABASE_PORT: 5432 + DATABASE_NAME: platform_api + DATABASE_USERNAME: postgres + DATABASE_PASSWORD: postgres + DATABASE_MAX_CONNECTIONS: 5 + DATABASE_TLS_ENABLED: "false" + MODEL_DISCOVERY_SERVER_URL: ${{ secrets.MODEL_DISCOVERY_SERVER_URL }} + MODEL_DISCOVERY_API_KEY: ${{ secrets.MODEL_DISCOVERY_API_KEY }} + AUTH_ENCODING_KEY: ${{ secrets.AUTH_ENCODING_KEY }} + AUTH_ADMIN_DOMAINS: ${{ secrets.AUTH_ADMIN_DOMAINS }} + BRAVE_SEARCH_PRO_API_KEY: ${{ secrets.BRAVE_SEARCH_PRO_API_KEY }} + TEST_TEMPLATE_DATABASE_NAME: platform_api_test_template + RUST_LOG: debug + DEV: "true" + + - name: Cleanup orphaned test databases + if: always() + continue-on-error: true + run: cargo test --test cleanup_test_databases -- --ignored --nocapture + env: + DATABASE_HOST: localhost + DATABASE_PORT: 5432 + DATABASE_USERNAME: postgres + DATABASE_PASSWORD: postgres + TEST_TEMPLATE_DATABASE_NAME: platform_api_test_template diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29e31ce9..d1a72694 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,7 +76,7 @@ jobs: # Template DB is created automatically on first test run with all migrations applied # Each test gets its own database copy for complete isolation - name: Run integration tests - run: cargo nextest run -p api --tests + run: cargo nextest run -p api --tests -E 'not test(/^real_test_/)' env: POSTGRES_PRIMARY_APP_ID: ${{ secrets.POSTGRES_PRIMARY_APP_ID }} DATABASE_HOST: localhost diff --git a/Makefile b/Makefile index ae67fb15..1c85fb0f 100644 --- a/Makefile +++ b/Makefile @@ -63,11 +63,15 @@ test: test-unit test-integration test-unit: @echo "Running unit tests..." - cargo test --lib --bins + cargo nextest run --lib --bins test-integration: @echo "Running integration/e2e tests..." - cargo test --test '*' + cargo nextest run -p api --tests -E 'not test(/^real_test_/)' + +test-integration-real: + @echo "Running integration/real_e2e tests..." + cargo nextest run -p api --tests -E 'test(/^real_test_/)' lint: @echo "Running clippy linter (strict mode)..." diff --git a/crates/api/tests/common/endpoints.rs b/crates/api/tests/common/endpoints.rs new file mode 100644 index 00000000..67eb122e --- /dev/null +++ b/crates/api/tests/common/endpoints.rs @@ -0,0 +1,536 @@ +//! Common HTTP-level helpers for API E2E tests. + +use api::models::{ConversationItemList, ConversationObject, ResponseObject}; +use api::routes::attestation::{AttestationResponse, SignatureResponse}; +use inference_providers::StreamChunk; +use serde_json::json; + +/// POST `/v1/chat/completions` and return the raw response body text. +/// +/// This is intentionally low-level so tests can compute hashes over the exact body when needed. +pub async fn post_chat_completions_raw( + server: &axum_test::TestServer, + api_key: &str, + request_body: &serde_json::Value, +) -> String { + let response = server + .post("/v1/chat/completions") + .add_header("Authorization", format!("Bearer {api_key}")) + .json(request_body) + .await; + + assert_eq!( + response.status_code(), + 200, + "POST /v1/chat/completions should return 200, got {} body={}", + response.status_code(), + response.text() + ); + + response.text() +} + +/// Extract chat completion id from a `/v1/chat/completions` streaming (SSE) body. +pub fn extract_chat_id_from_chat_completions_sse(response_text: &str) -> Option { + for line in response_text.lines() { + let Some(data) = line.strip_prefix("data: ") else { + continue; + }; + + if data.trim() == "[DONE]" { + break; + } + + if let Ok(StreamChunk::Chat(chat_chunk)) = serde_json::from_str::(data) { + return Some(chat_chunk.id); + } + } + + None +} + +/// Run a `/v1/chat/completions` streaming request and return `(chat_id, raw_body)`. +pub async fn create_chat_completion_stream_and_get_id( + server: &axum_test::TestServer, + api_key: &str, + request_body: &serde_json::Value, +) -> (String, String) { + let response_text = post_chat_completions_raw(server, api_key, request_body).await; + let chat_id = extract_chat_id_from_chat_completions_sse(&response_text) + .expect("Should extract chat_id from SSE stream"); + (chat_id, response_text) +} + +/// Run a non-streaming `/v1/chat/completions` request and return `(chat_id, raw_body)`. +pub async fn create_chat_completion_non_stream_and_get_id( + server: &axum_test::TestServer, + api_key: &str, + request_body: &serde_json::Value, +) -> (String, String) { + let response_text = post_chat_completions_raw(server, api_key, request_body).await; + let response_json: serde_json::Value = + serde_json::from_str(&response_text).expect("Chat completion response must be valid JSON"); + let chat_id = response_json + .get("id") + .and_then(|v| v.as_str()) + .expect("Chat completion response must have id field") + .to_string(); + (chat_id, response_text) +} + +/// POST `/v1/responses` (without conversation) and return the raw response body text. +/// +/// This matches the real-e2e tests that only provide `input`/`model` fields. +pub async fn post_responses_raw( + server: &axum_test::TestServer, + api_key: &str, + request_body: &serde_json::Value, +) -> String { + let response = server + .post("/v1/responses") + .add_header("Authorization", format!("Bearer {api_key}")) + .json(request_body) + .await; + + assert_eq!( + response.status_code(), + 200, + "POST /v1/responses should return 200, got {} body={}", + response.status_code(), + response.text() + ); + + response.text() +} + +/// Extract response id from a `/v1/responses` streaming (SSE) body by looking for `response.created`. +pub fn extract_response_id_from_responses_sse(response_text: &str) -> Option { + for line_chunk in response_text.split("\n\n") { + if line_chunk.trim().is_empty() { + continue; + } + + let mut event_type = ""; + let mut event_data = ""; + + for line in line_chunk.lines() { + if let Some(event_name) = line.strip_prefix("event: ") { + event_type = event_name; + } else if let Some(data) = line.strip_prefix("data: ") { + event_data = data; + } + } + + if event_type != "response.created" || event_data.is_empty() { + continue; + } + + let Ok(event_json) = serde_json::from_str::(event_data) else { + continue; + }; + + if let Some(response_obj) = event_json.get("response") { + if let Some(id) = response_obj.get("id").and_then(|v| v.as_str()) { + return Some(id.to_string()); + } + } + } + + None +} + +/// Run a `/v1/responses` streaming request (without conversation) and return `(response_id, raw_body)`. +pub async fn create_response_stream_no_conversation_and_get_id( + server: &axum_test::TestServer, + api_key: &str, + request_body: &serde_json::Value, +) -> (String, String) { + let response_text = post_responses_raw(server, api_key, request_body).await; + let response_id = extract_response_id_from_responses_sse(&response_text) + .expect("Should have extracted response_id from stream"); + (response_id, response_text) +} + +/// Run a non-streaming `/v1/responses` request (without conversation) and return `(response_id, raw_body)`. +pub async fn create_response_non_stream_no_conversation_and_get_id( + server: &axum_test::TestServer, + api_key: &str, + request_body: &serde_json::Value, +) -> (String, String) { + let response_text = post_responses_raw(server, api_key, request_body).await; + let response_json: serde_json::Value = + serde_json::from_str(&response_text).expect("Response must be valid JSON"); + let response_id = response_json + .get("id") + .and_then(|v| v.as_str()) + .expect("Response must have id field") + .to_string(); + (response_id, response_text) +} + +pub async fn create_conversation( + server: &axum_test::TestServer, + api_key: String, +) -> ConversationObject { + create_conversation_with_metadata( + server, + api_key, + Some(json!({"name": "Test Conversation", "description": "A test conversation"})), + ) + .await +} + +pub async fn create_conversation_with_metadata( + server: &axum_test::TestServer, + api_key: String, + metadata: Option, +) -> ConversationObject { + let response = server + .post("/v1/conversations") + .add_header("Authorization", format!("Bearer {api_key}")) + .json(&serde_json::json!({ + "metadata": metadata.unwrap_or_else(|| serde_json::json!({})) + })) + .await; + + assert_eq!( + response.status_code(), + 201, + "Create conversation should return 201, got {} body={}", + response.status_code(), + response.text() + ); + response.json::() +} + +pub async fn get_conversation( + server: &axum_test::TestServer, + conversation_id: String, + api_key: String, +) -> ConversationObject { + let response = server + .get(format!("/v1/conversations/{conversation_id}").as_str()) + .add_header("Authorization", format!("Bearer {api_key}")) + .await; + + assert_eq!( + response.status_code(), + 200, + "Get conversation should return 200, got {} body={}", + response.status_code(), + response.text() + ); + response.json::() +} + +pub async fn list_conversation_items( + server: &axum_test::TestServer, + conversation_id: String, + api_key: String, +) -> ConversationItemList { + let response = server + .get(format!("/v1/conversations/{conversation_id}/items").as_str()) + .add_header("Authorization", format!("Bearer {api_key}")) + .await; + + assert_eq!( + response.status_code(), + 200, + "List conversation items should return 200, got {} body={}", + response.status_code(), + response.text() + ); + response.json::() +} + +/// Update conversation metadata by POST to `/v1/conversations/{id}`. +/// +/// This replaces the entire metadata object with the provided one. +pub async fn update_conversation_metadata( + server: &axum_test::TestServer, + conversation_id: String, + api_key: String, + metadata: serde_json::Value, +) -> ConversationObject { + let response = server + .post(format!("/v1/conversations/{conversation_id}").as_str()) + .add_header("Authorization", format!("Bearer {api_key}")) + .json(&serde_json::json!({ + "metadata": metadata + })) + .await; + + assert_eq!( + response.status_code(), + 200, + "Update conversation metadata should return 200, got {} body={}", + response.status_code(), + response.text() + ); + response.json::() +} + +/// Delete a conversation by DELETE to `/v1/conversations/{id}`. +pub async fn delete_conversation( + server: &axum_test::TestServer, + conversation_id: String, + api_key: String, +) { + let response = server + .delete(format!("/v1/conversations/{conversation_id}").as_str()) + .add_header("Authorization", format!("Bearer {api_key}")) + .await; + + assert_eq!( + response.status_code(), + 200, + "Delete conversation should return 200, got {} body={}", + response.status_code(), + response.text() + ); +} + +pub async fn create_response( + server: &axum_test::TestServer, + conversation_id: String, + model: String, + message: String, + max_tokens: i64, + api_key: String, +) -> ResponseObject { + create_response_with_temperature( + server, + conversation_id, + model, + message, + max_tokens, + api_key, + 0.7, + ) + .await +} + +pub async fn create_response_with_temperature( + server: &axum_test::TestServer, + conversation_id: String, + model: String, + message: String, + max_tokens: i64, + api_key: String, + temperature: f64, +) -> ResponseObject { + let response = server + .post("/v1/responses") + .add_header("Authorization", format!("Bearer {api_key}")) + .json(&serde_json::json!({ + "conversation": { + "id": conversation_id, + }, + "input": message, + "temperature": temperature, + "max_output_tokens": max_tokens, + "stream": false, + "model": model + })) + .await; + + assert_eq!( + response.status_code(), + 200, + "Create response should return 200, got {} body={}", + response.status_code(), + response.text() + ); + response.json::() +} + +pub async fn create_response_stream( + server: &axum_test::TestServer, + conversation_id: String, + model: String, + message: String, + max_tokens: i64, + api_key: String, +) -> (String, ResponseObject) { + create_response_stream_with_temperature( + server, + conversation_id, + model, + message, + max_tokens, + api_key, + 0.7, + ) + .await +} + +pub async fn create_response_stream_with_temperature( + server: &axum_test::TestServer, + conversation_id: String, + model: String, + message: String, + max_tokens: i64, + api_key: String, + temperature: f64, +) -> (String, ResponseObject) { + let response = server + .post("/v1/responses") + .add_header("Authorization", format!("Bearer {api_key}")) + .json(&serde_json::json!({ + "conversation": { + "id": conversation_id, + }, + "input": message, + "temperature": temperature, + "max_output_tokens": max_tokens, + "stream": true, + "model": model + })) + .await; + + assert_eq!( + response.status_code(), + 200, + "Create streaming response should return 200, got {} body={}", + response.status_code(), + response.text() + ); + + // For streaming responses, we get SSE events as text: "event: \ndata: \n\n" + let response_text = response.text(); + + let mut content = String::new(); + let mut final_response: Option = None; + + for line_chunk in response_text.split("\n\n") { + if line_chunk.trim().is_empty() { + continue; + } + + let mut event_type = ""; + let mut event_data = ""; + + for line in line_chunk.lines() { + if let Some(event_name) = line.strip_prefix("event: ") { + event_type = event_name; + } else if let Some(data) = line.strip_prefix("data: ") { + event_data = data; + } + } + + if event_data.is_empty() { + continue; + } + + let Ok(event_json) = serde_json::from_str::(event_data) else { + continue; + }; + + match event_type { + "response.output_text.delta" => { + if let Some(delta) = event_json.get("delta").and_then(|v| v.as_str()) { + content.push_str(delta); + } + } + "response.completed" => { + if let Some(response_obj) = event_json.get("response") { + final_response = Some( + serde_json::from_value::(response_obj.clone()) + .expect("Failed to parse response.completed event"), + ); + } + } + _ => {} + } + } + + let final_resp = final_response.expect("Expected response.completed event from stream"); + (content, final_resp) +} + +pub async fn upload_file( + server: &axum_test::TestServer, + api_key: &str, + filename: &str, + body: &[u8], + mimetype: &str, + purpose: &str, +) -> axum_test::TestResponse { + server + .post("/v1/files") + .add_header("Authorization", format!("Bearer {api_key}")) + .multipart( + axum_test::multipart::MultipartForm::new() + .add_text("purpose", purpose) + .add_part( + "file", + axum_test::multipart::Part::bytes(body.to_vec()) + .file_name(filename) + .mime_type(mimetype), + ), + ) + .await +} + +/// GET `/v1/signature/{id}` for a given model and signing algorithm. +/// +/// This helper is used by real_e2e signature tests. It ensures that the `model` +/// query parameter is URL-encoded before sending the request. +pub async fn get_signature_for_id( + server: &axum_test::TestServer, + api_key: &str, + id: &str, + model_name: &str, + signing_algo: &str, +) -> SignatureResponse { + let encoded_model = + url::form_urlencoded::byte_serialize(model_name.as_bytes()).collect::(); + let signature_url = + format!("/v1/signature/{id}?model={encoded_model}&signing_algo={signing_algo}"); + + let response = server + .get(&signature_url) + .add_header("Authorization", format!("Bearer {api_key}")) + .await; + + assert_eq!( + response.status_code(), + 200, + "Signature endpoint should return success, got {} body={}", + response.status_code(), + response.text() + ); + + response.json::() +} + +/// GET `/v1/attestation/report` for a given model and signing algorithm. +/// +/// This helper is used by real_e2e signature tests. It ensures that the `model` +/// query parameter is URL-encoded before sending the request. +pub async fn get_attestation_report( + server: &axum_test::TestServer, + api_key: &str, + model_name: &str, + signing_algo: &str, +) -> AttestationResponse { + let encoded_model = + url::form_urlencoded::byte_serialize(model_name.as_bytes()).collect::(); + let attestation_url = + format!("/v1/attestation/report?model={encoded_model}&signing_algo={signing_algo}"); + + let response = server + .get(&attestation_url) + .add_header("Authorization", format!("Bearer {api_key}")) + .await; + + assert_eq!( + response.status_code(), + 200, + "Attestation report should return successfully, got {} body={}", + response.status_code(), + response.text() + ); + + response.json::() +} diff --git a/crates/api/tests/common/mod.rs b/crates/api/tests/common/mod.rs index 6b9a0568..896bf9b0 100644 --- a/crates/api/tests/common/mod.rs +++ b/crates/api/tests/common/mod.rs @@ -5,6 +5,7 @@ //! each test clones from template for isolation and speed. pub mod db_setup; +pub mod endpoints; use api::{build_app_with_config, init_auth_services, models::BatchUpdateModelApiRequest}; use base64::Engine; diff --git a/crates/api/tests/e2e_conversations.rs b/crates/api/tests/e2e_conversations.rs index 28449d94..4fbc6da5 100644 --- a/crates/api/tests/e2e_conversations.rs +++ b/crates/api/tests/e2e_conversations.rs @@ -3,169 +3,11 @@ mod common; use common::*; +use crate::common::endpoints::*; use api::models::{ ConversationContentPart, ConversationItem, ResponseOutputContent, ResponseOutputItem, }; -// Helper functions for conversation and response tests -async fn create_conversation( - server: &axum_test::TestServer, - api_key: String, -) -> api::models::ConversationObject { - let response = server - .post("/v1/conversations") - .add_header("Authorization", format!("Bearer {api_key}")) - .json(&serde_json::json!({ - "name": "Test Conversation", - "description": "A test conversation" - })) - .await; - assert_eq!(response.status_code(), 201); - response.json::() -} - -#[allow(dead_code)] -async fn get_conversation( - server: &axum_test::TestServer, - conversation_id: String, - api_key: String, -) -> api::models::ConversationObject { - let response = server - .get(format!("/v1/conversations/{conversation_id}").as_str()) - .add_header("Authorization", format!("Bearer {api_key}")) - .await; - assert_eq!(response.status_code(), 200); - response.json::() -} - -async fn list_conversation_items( - server: &axum_test::TestServer, - conversation_id: String, - api_key: String, -) -> api::models::ConversationItemList { - let response = server - .get(format!("/v1/conversations/{conversation_id}/items").as_str()) - .add_header("Authorization", format!("Bearer {api_key}")) - .await; - assert_eq!(response.status_code(), 200); - response.json::() -} - -async fn create_response( - server: &axum_test::TestServer, - conversation_id: String, - model: String, - message: String, - max_tokens: i64, - api_key: String, -) -> api::models::ResponseObject { - let response = server - .post("/v1/responses") - .add_header("Authorization", format!("Bearer {api_key}")) - .json(&serde_json::json!({ - "conversation": { - "id": conversation_id, - }, - "input": message, - "temperature": 0.7, - "max_output_tokens": max_tokens, - "stream": false, - "model": model - })) - .await; - assert_eq!(response.status_code(), 200); - response.json::() -} - -async fn create_response_stream( - server: &axum_test::TestServer, - conversation_id: String, - model: String, - message: String, - max_tokens: i64, - api_key: String, -) -> (String, api::models::ResponseObject) { - let response = server - .post("/v1/responses") - .add_header("Authorization", format!("Bearer {api_key}")) - .json(&serde_json::json!({ - "conversation": { - "id": conversation_id, - }, - "input": message, - "temperature": 0.7, - "max_output_tokens": max_tokens, - "stream": true, - "model": model - })) - .await; - - assert_eq!(response.status_code(), 200); - - // For streaming responses, we get SSE events as text - let response_text = response.text(); - - let mut content = String::new(); - let mut final_response: Option = None; - - // Parse SSE format: "event: \ndata: \n\n" - for line_chunk in response_text.split("\n\n") { - if line_chunk.trim().is_empty() { - continue; - } - - let mut event_type = ""; - let mut event_data = ""; - - for line in line_chunk.lines() { - if let Some(event_name) = line.strip_prefix("event: ") { - event_type = event_name; - } else if let Some(data) = line.strip_prefix("data: ") { - event_data = data; - } - } - - if !event_data.is_empty() { - if let Ok(event_json) = serde_json::from_str::(event_data) { - match event_type { - "response.output_text.delta" => { - // Accumulate content deltas as they arrive - if let Some(delta) = event_json.get("delta").and_then(|v| v.as_str()) { - content.push_str(delta); - println!("Delta: {delta}"); - } - } - "response.completed" => { - // Extract final response from completed event - if let Some(response_obj) = event_json.get("response") { - final_response = Some( - serde_json::from_value::( - response_obj.clone(), - ) - .expect("Failed to parse response.completed event"), - ); - println!("Stream completed"); - } - } - "response.created" => { - println!("Response created"); - } - "response.in_progress" => { - println!("Response in progress"); - } - _ => { - println!("Event: {event_type}"); - } - } - } - } - } - - let final_resp = - final_response.expect("Expected to receive response.completed event from stream"); - (content, final_resp) -} - // ============================================ // Response Tests // ============================================ diff --git a/crates/api/tests/e2e_files.rs b/crates/api/tests/e2e_files.rs index 497d0b80..83de0817 100644 --- a/crates/api/tests/e2e_files.rs +++ b/crates/api/tests/e2e_files.rs @@ -1,34 +1,10 @@ // Import common test utilities mod common; +use common::endpoints::upload_file; use common::*; use services::id_prefixes::PREFIX_FILE; -/// Helper function to upload a file -async fn upload_file( - server: &axum_test::TestServer, - api_key: &str, - filename: &str, - content: &[u8], - content_type: &str, - purpose: &str, -) -> axum_test::TestResponse { - server - .post("/v1/files") - .add_header("Authorization", format!("Bearer {api_key}")) - .multipart( - axum_test::multipart::MultipartForm::new() - .add_text("purpose", purpose) - .add_part( - "file", - axum_test::multipart::Part::bytes(content.to_vec()) - .file_name(filename) - .mime_type(content_type), - ), - ) - .await -} - /// Helper function to upload a file with expiration async fn upload_file_with_expiration( server: &axum_test::TestServer, diff --git a/crates/api/tests/real_e2e_conversations.rs b/crates/api/tests/real_e2e_conversations.rs new file mode 100644 index 00000000..8db2e437 --- /dev/null +++ b/crates/api/tests/real_e2e_conversations.rs @@ -0,0 +1,195 @@ +mod common; + +use common::endpoints; +use common::*; + +use api::models::{ConversationContentPart, ConversationItem}; + +#[tokio::test] +async fn real_test_conversation_items_populated_by_responses_non_stream() { + let (server, _pool, _db, _guard) = setup_test_server_real_providers().await; + + // Seed model metadata/pricing into DB so /v1/responses passes validation/usage logic. + let model_name = setup_qwen_model(&server).await; + let org = setup_org_with_credits(&server, 10_000_000_000i64).await; // $10.00 USD + let api_key = get_api_key_for_org(&server, org.id).await; + + let conversation = endpoints::create_conversation_with_metadata( + &server, + api_key.clone(), + Some(serde_json::json!({ "source": "real_e2e" })), + ) + .await; + let message = "Reply with exactly the word: ok".to_string(); + + let resp = endpoints::create_response_with_temperature( + &server, + conversation.id.clone(), + model_name, + message.clone(), + 32, + api_key.clone(), + 0.2, + ) + .await; + + assert_eq!(resp.status, api::models::ResponseStatus::Completed); + + let items = endpoints::list_conversation_items(&server, conversation.id, api_key).await; + + // We expect at least the user message to be persisted; assistant may include extra items depending on provider. + assert!( + items.data.iter().any(|it| match it { + ConversationItem::Message { role, content, .. } if role == "user" => + content.iter().any(|p| { + matches!(p, ConversationContentPart::InputText { text } if text == &message) + }), + _ => false, + }), + "Expected conversation items to include the user input message" + ); + + assert!( + items.data.iter().any(|it| match it { + ConversationItem::Message { role, .. } => role == "assistant", + _ => false, + }), + "Expected at least one assistant message item in conversation" + ); +} + +#[tokio::test] +async fn real_test_conversation_items_populated_by_responses_streaming() { + let (server, _pool, _db, _guard) = setup_test_server_real_providers().await; + + // Seed model metadata/pricing into DB so /v1/responses passes validation/usage logic. + let model_name = setup_qwen_model(&server).await; + let org = setup_org_with_credits(&server, 10_000_000_000i64).await; // $10.00 USD + let api_key = get_api_key_for_org(&server, org.id).await; + + let conversation = endpoints::create_conversation_with_metadata( + &server, + api_key.clone(), + Some(serde_json::json!({ "source": "real_e2e" })), + ) + .await; + let message = "Write a one-word reply: ok".to_string(); + + let (_streamed_text, final_resp) = endpoints::create_response_stream_with_temperature( + &server, + conversation.id.clone(), + model_name, + message.clone(), + 64, + api_key.clone(), + 0.2, + ) + .await; + + assert_eq!(final_resp.status, api::models::ResponseStatus::Completed); + + let items = endpoints::list_conversation_items(&server, conversation.id, api_key).await; + assert!( + !items.data.is_empty(), + "Expected conversation items to be present after streaming response" + ); + + assert!( + items.data.iter().any(|it| match it { + ConversationItem::Message { role, content, .. } if role == "user" => + content.iter().any(|p| { + matches!(p, ConversationContentPart::InputText { text } if text == &message) + }), + _ => false, + }), + "Expected conversation items to include the user input message" + ); +} + +#[tokio::test] +async fn real_test_update_conversation_metadata() { + let (server, _pool, _db, _guard) = setup_test_server_real_providers().await; + let _model_name = setup_qwen_model(&server).await; + let org = setup_org_with_credits(&server, 10_000_000_000i64).await; + let api_key = get_api_key_for_org(&server, org.id).await; + + let conversation = endpoints::create_conversation_with_metadata( + &server, + api_key.clone(), + Some(serde_json::json!({ + "title": "Original Title", + "description": "Should be removed", + "source": "real_e2e" + })), + ) + .await; + + let updated_conv = endpoints::update_conversation_metadata( + &server, + conversation.id.clone(), + api_key.clone(), + serde_json::json!({ + "title": "Updated Title", + "context": "full replacement test" + }), + ) + .await; + let metadata_obj = updated_conv + .metadata + .as_object() + .expect("Metadata should be an object"); + assert_eq!( + metadata_obj.len(), + 2, + "Metadata should only contain the new keys" + ); + assert_eq!( + metadata_obj + .get("title") + .and_then(|v| v.as_str()) + .unwrap_or_default(), + "Updated Title" + ); + assert_eq!( + metadata_obj + .get("context") + .and_then(|v| v.as_str()) + .unwrap_or_default(), + "full replacement test" + ); + assert!( + metadata_obj.get("description").is_none(), + "Old metadata keys should be removed when updating" + ); +} + +#[tokio::test] +async fn real_test_delete_conversation() { + let (server, _pool, _db, _guard) = setup_test_server_real_providers().await; + let _model_name = setup_qwen_model(&server).await; + let org = setup_org_with_credits(&server, 10_000_000_000i64).await; + let api_key = get_api_key_for_org(&server, org.id).await; + + let conversation = endpoints::create_conversation_with_metadata( + &server, + api_key.clone(), + Some(serde_json::json!({ "title": "To be deleted" })), + ) + .await; + + let _pre_delete = + endpoints::get_conversation(&server, conversation.id.clone(), api_key.clone()).await; + + endpoints::delete_conversation(&server, conversation.id.clone(), api_key.clone()).await; + + let get_after_delete = server + .get(format!("/v1/conversations/{}", conversation.id).as_str()) + .add_header("Authorization", format!("Bearer {api_key}")) + .await; + + assert_eq!( + get_after_delete.status_code(), + 404, + "Deleted conversation should no longer be accessible" + ); +} diff --git a/crates/api/tests/real_e2e_response_signature_attestation.rs b/crates/api/tests/real_e2e_response_signature_attestation.rs new file mode 100644 index 00000000..2e3763af --- /dev/null +++ b/crates/api/tests/real_e2e_response_signature_attestation.rs @@ -0,0 +1,92 @@ +// Real provider integration test verifying the response signature matches gateway attestation metadata. +mod common; + +use common::*; +use endpoints::*; + +#[tokio::test] +async fn real_test_signature_signing_address_matches_gateway_attestation_stream() { + let (server, _pool, _db, _guard) = setup_test_server_real_providers().await; + let model_name = setup_qwen_model(&server).await; + let org = setup_org_with_credits(&server, 10000000000i64).await; + let api_key = get_api_key_for_org(&server, org.id).await; + + let request_body = serde_json::json!({ + "input": "Respond with only two words.", + "temperature": 0.7, + "max_output_tokens": 50, + "stream": true, + "model": model_name, + }); + let (response_id, _raw_body) = + create_response_stream_no_conversation_and_get_id(&server, &api_key, &request_body).await; + + tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; + + let signature = + get_signature_for_id(&server, &api_key, &response_id, &model_name, "ecdsa").await; + let signing_address = signature.signing_address; + assert!( + !signing_address.is_empty(), + "Signing address should not be empty" + ); + + let attestation = get_attestation_report(&server, &api_key, &model_name, "ecdsa").await; + let gateway_address = attestation.gateway_attestation.signing_address; + assert!( + !gateway_address.is_empty(), + "Gateway attestation should expose signing_address" + ); + + let normalized_signature_address = signing_address.trim_start_matches("0x").to_lowercase(); + let normalized_gateway_address = gateway_address.trim_start_matches("0x").to_lowercase(); + + assert_eq!( + normalized_signature_address, normalized_gateway_address, + "Signature signing address {signing_address} should match gateway attestation signing address {gateway_address}" + ); +} + +#[tokio::test] +async fn real_test_signature_signing_address_matches_gateway_attestation_non_stream() { + let (server, _pool, _db, _guard) = setup_test_server_real_providers().await; + let model_name = setup_qwen_model(&server).await; + let org = setup_org_with_credits(&server, 10000000000i64).await; + let api_key = get_api_key_for_org(&server, org.id).await; + + let request_body = serde_json::json!({ + "input": "Respond with only two words.", + "temperature": 0.7, + "max_output_tokens": 50, + "stream": false, + "model": model_name, + }); + let (response_id, _raw_body) = + create_response_non_stream_no_conversation_and_get_id(&server, &api_key, &request_body) + .await; + + tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; + + let signature = + get_signature_for_id(&server, &api_key, &response_id, &model_name, "ecdsa").await; + let signing_address = signature.signing_address; + assert!( + !signing_address.is_empty(), + "Signing address should not be empty" + ); + + let attestation = get_attestation_report(&server, &api_key, &model_name, "ecdsa").await; + let gateway_address = attestation.gateway_attestation.signing_address; + assert!( + !gateway_address.is_empty(), + "Gateway attestation should expose signing_address" + ); + + let normalized_signature_address = signing_address.trim_start_matches("0x").to_lowercase(); + let normalized_gateway_address = gateway_address.trim_start_matches("0x").to_lowercase(); + + assert_eq!( + normalized_signature_address, normalized_gateway_address, + "Signature signing address {signing_address} should match gateway attestation signing address {gateway_address}" + ); +} diff --git a/crates/api/tests/real_e2e_signature_verification.rs b/crates/api/tests/real_e2e_signature_verification.rs new file mode 100644 index 00000000..8e7bd644 --- /dev/null +++ b/crates/api/tests/real_e2e_signature_verification.rs @@ -0,0 +1,126 @@ +mod common; + +use common::*; +use endpoints::*; + +#[tokio::test] +async fn real_test_signature_signing_address_matches_model_attestation_stream() { + let (server, _pool, _db, _guard) = setup_test_server_real_providers().await; + let model_name = setup_qwen_model(&server).await; + let org = setup_org_with_credits(&server, 10000000000i64).await; + let api_key = get_api_key_for_org(&server, org.id).await; + + let request_body = serde_json::json!({ + "messages": [ + { + "role": "user", + "content": "Respond with a short sentence." + } + ], + "stream": true, + "model": model_name, + }); + + let (chat_id, _raw_body) = + create_chat_completion_stream_and_get_id(&server, &api_key, &request_body).await; + + tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; + + let signature = get_signature_for_id(&server, &api_key, &chat_id, &model_name, "ecdsa").await; + let signing_address = signature.signing_address; + assert!( + !signing_address.is_empty(), + "Signing address should not be empty" + ); + + let attestation = get_attestation_report(&server, &api_key, &model_name, "ecdsa").await; + let attestation_addresses: Vec = attestation + .model_attestations + .iter() + .filter_map(|attestation| { + attestation + .get("signing_address") + .and_then(|value| value.as_str()) + .map(|value| value.to_string()) + }) + .collect(); + + assert!( + !attestation_addresses.is_empty(), + "Model attestation list must contain at least one signing_address" + ); + + let normalized_signature_address = signing_address.trim_start_matches("0x").to_lowercase(); + let normalized_attestation_addresses: Vec = attestation_addresses + .iter() + .map(|addr| addr.trim_start_matches("0x").to_lowercase()) + .collect(); + + assert!( + normalized_attestation_addresses + .iter() + .any(|addr| addr == &normalized_signature_address), + "Signing address {signing_address} was not found in the model attestation list: {attestation_addresses:?}" + ); +} + +#[tokio::test] +async fn real_test_signature_signing_address_matches_model_attestation_non_stream() { + let (server, _pool, _db, _guard) = setup_test_server_real_providers().await; + let model_name = setup_qwen_model(&server).await; + let org = setup_org_with_credits(&server, 10000000000i64).await; + let api_key = get_api_key_for_org(&server, org.id).await; + + let request_body = serde_json::json!({ + "messages": [ + { + "role": "user", + "content": "Respond with a short sentence." + } + ], + "stream": false, + "model": model_name, + }); + + let (chat_id, _raw_body) = + create_chat_completion_non_stream_and_get_id(&server, &api_key, &request_body).await; + + tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; + + let signature = get_signature_for_id(&server, &api_key, &chat_id, &model_name, "ecdsa").await; + let signing_address = signature.signing_address; + assert!( + !signing_address.is_empty(), + "Signing address should not be empty" + ); + + let attestation = get_attestation_report(&server, &api_key, &model_name, "ecdsa").await; + let attestation_addresses: Vec = attestation + .model_attestations + .iter() + .filter_map(|attestation| { + attestation + .get("signing_address") + .and_then(|value| value.as_str()) + .map(|value| value.to_string()) + }) + .collect(); + + assert!( + !attestation_addresses.is_empty(), + "Model attestation list must contain at least one signing_address" + ); + + let normalized_signature_address = signing_address.trim_start_matches("0x").to_lowercase(); + let normalized_attestation_addresses: Vec = attestation_addresses + .iter() + .map(|addr| addr.trim_start_matches("0x").to_lowercase()) + .collect(); + + assert!( + normalized_attestation_addresses + .iter() + .any(|addr| addr == &normalized_signature_address), + "Signing address {signing_address} was not found in the model attestation list: {attestation_addresses:?}" + ); +}