Test Suite #139
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Suite | |
| on: | |
| push: | |
| branches: [main, develop] | |
| pull_request: | |
| branches: [main, develop] | |
| schedule: | |
| # Run tests daily at 2 AM UTC | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| coverage: | |
| description: 'Generate coverage report' | |
| required: false | |
| type: boolean | |
| default: false | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: 1 | |
| RUSTFLAGS: -D warnings | |
| jobs: | |
| # Fast checks that run on every commit | |
| fast-checks: | |
| name: Fast Checks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt, clippy | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: fast-checks-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Check formatting | |
| run: cargo fmt --all -- --check | |
| - name: Run clippy | |
| run: cargo clippy --workspace --all-features --all-targets -- -D warnings | |
| - name: Check documentation | |
| run: cargo doc --workspace --all-features --no-deps | |
| env: | |
| RUSTDOCFLAGS: -D warnings | |
| # Unit tests - fast, no external dependencies | |
| unit-tests: | |
| name: Unit Tests | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 20 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| rust: [stable, beta] | |
| exclude: | |
| # Skip beta on macOS and Windows to reduce CI time | |
| - os: macos-latest | |
| rust: beta | |
| - os: windows-latest | |
| rust: beta | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust toolchain | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: ${{ matrix.rust }} | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: unit-${{ matrix.os }}-${{ matrix.rust }}-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install cargo-nextest | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: cargo-nextest | |
| - name: Run unit tests | |
| run: cargo nextest run --workspace --lib --all-features --profile ci | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: unit-test-results-${{ matrix.os }}-${{ matrix.rust }} | |
| path: target/nextest/ci/junit.xml | |
| if-no-files-found: ignore | |
| # Integration tests - require Docker services | |
| integration-tests: | |
| name: Integration Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| services: | |
| postgres: | |
| image: timescale/timescaledb:2.14.2-pg16 | |
| env: | |
| POSTGRES_USER: test_user | |
| POSTGRES_PASSWORD: test_password | |
| POSTGRES_DB: llm_observatory_test | |
| TIMESCALEDB_TELEMETRY: off | |
| options: >- | |
| --health-cmd "pg_isready -U test_user -d llm_observatory_test" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| redis: | |
| image: redis:7.2-alpine | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 6379:6379 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: integration-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install cargo-nextest | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: cargo-nextest | |
| - name: Install sqlx-cli | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: sqlx-cli | |
| - name: Run database migrations | |
| run: | | |
| if [ -d migrations ]; then | |
| sqlx migrate run --source migrations | |
| fi | |
| env: | |
| DATABASE_URL: postgres://test_user:test_password@localhost:5432/llm_observatory_test | |
| - name: Run integration tests | |
| run: cargo nextest run --workspace --test '*' --all-features --profile ci | |
| env: | |
| DATABASE_URL: postgres://test_user:test_password@localhost:5432/llm_observatory_test | |
| REDIS_URL: redis://localhost:6379 | |
| RUST_LOG: info,sqlx=warn | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: integration-test-results | |
| path: target/nextest/ci/junit.xml | |
| if-no-files-found: ignore | |
| # Parallel test execution using matrix strategy | |
| parallel-tests: | |
| name: Parallel Tests (Shard ${{ matrix.shard }}) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1, 2, 3, 4] | |
| services: | |
| postgres: | |
| image: timescale/timescaledb:2.14.2-pg16 | |
| env: | |
| POSTGRES_USER: test_user | |
| POSTGRES_PASSWORD: test_password | |
| POSTGRES_DB: llm_observatory_test | |
| options: >- | |
| --health-cmd "pg_isready -U test_user" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| redis: | |
| image: redis:7.2-alpine | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 6379:6379 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: parallel-${{ matrix.shard }}-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install cargo-nextest | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: cargo-nextest | |
| - name: Run tests (shard ${{ matrix.shard }}/4) | |
| run: | | |
| cargo nextest run \ | |
| --workspace \ | |
| --all-features \ | |
| --profile ci \ | |
| --partition hash:${{ matrix.shard }}/4 | |
| env: | |
| DATABASE_URL: postgres://test_user:test_password@localhost:5432/llm_observatory_test | |
| REDIS_URL: redis://localhost:6379 | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: parallel-test-results-shard-${{ matrix.shard }} | |
| path: target/nextest/ci/junit.xml | |
| if-no-files-found: ignore | |
| # Code coverage using Docker | |
| coverage: | |
| name: Code Coverage | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| if: github.event_name == 'pull_request' || github.event.inputs.coverage == 'true' || github.ref == 'refs/heads/main' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Start test services | |
| run: | | |
| docker compose -f docker-compose.test.yml up -d timescaledb-test redis-test | |
| docker compose -f docker-compose.test.yml ps | |
| - name: Wait for services | |
| run: | | |
| timeout 60 bash -c 'until docker compose -f docker-compose.test.yml exec -T timescaledb-test pg_isready -U test_user; do sleep 2; done' | |
| timeout 60 bash -c 'until docker compose -f docker-compose.test.yml exec -T redis-test redis-cli ping; do sleep 2; done' | |
| - name: Run coverage | |
| run: | | |
| docker compose -f docker-compose.test.yml run --rm coverage-runner | |
| - name: Copy coverage results | |
| run: | | |
| docker compose -f docker-compose.test.yml cp coverage-runner:/workspace/coverage ./coverage | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: ./coverage/lcov.info | |
| flags: unittests | |
| name: codecov-llm-observatory | |
| fail_ci_if_error: false | |
| - name: Upload coverage artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report | |
| path: coverage/ | |
| if-no-files-found: warn | |
| - name: Cleanup | |
| if: always() | |
| run: docker compose -f docker-compose.test.yml down -v | |
| # Docker-based test execution | |
| docker-tests: | |
| name: Docker Test Suite | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build test image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: docker/Dockerfile.test | |
| target: test-runner | |
| tags: llm-observatory-test:latest | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| load: true | |
| - name: Start test environment | |
| run: | | |
| docker compose -f docker-compose.test.yml up -d | |
| docker compose -f docker-compose.test.yml ps | |
| - name: Run all tests | |
| run: | | |
| docker compose -f docker-compose.test.yml run --rm test-runner | |
| - name: Copy test results | |
| if: always() | |
| run: | | |
| docker compose -f docker-compose.test.yml cp test-runner:/workspace/test-results ./test-results | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-test-results | |
| path: test-results/ | |
| if-no-files-found: warn | |
| - name: Cleanup | |
| if: always() | |
| run: docker compose -f docker-compose.test.yml down -v | |
| # Security audit | |
| security-audit: | |
| name: Security Audit | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Install cargo-audit | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: cargo-audit | |
| - name: Run security audit | |
| run: cargo audit --json > audit-report.json | |
| continue-on-error: true | |
| - name: Upload audit report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: security-audit-report | |
| path: audit-report.json | |
| if-no-files-found: warn | |
| # Aggregate test results | |
| test-summary: | |
| name: Test Summary | |
| runs-on: ubuntu-latest | |
| needs: [fast-checks, unit-tests, integration-tests, parallel-tests] | |
| if: always() | |
| steps: | |
| - name: Download all test results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: test-results/ | |
| - name: Generate summary | |
| run: | | |
| echo "# Test Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## Jobs Status" >> $GITHUB_STEP_SUMMARY | |
| echo "- Fast Checks: ${{ needs.fast-checks.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Unit Tests: ${{ needs.unit-tests.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Integration Tests: ${{ needs.integration-tests.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Parallel Tests: ${{ needs.parallel-tests.result }}" >> $GITHUB_STEP_SUMMARY | |
| - name: Check overall status | |
| if: | | |
| needs.fast-checks.result != 'success' || | |
| needs.unit-tests.result != 'success' || | |
| needs.integration-tests.result != 'success' || | |
| needs.parallel-tests.result != 'success' | |
| run: exit 1 |