Skip to content

Commit 093d78a

Browse files
dacharycdacharyc
committed
Add integration tests, update workflow and script
1 parent 65bf062 commit 093d78a

File tree

6 files changed

+838
-51
lines changed

6 files changed

+838
-51
lines changed

‎.github/scripts/generate-test-summary-pytest.sh‎

100644100755
Lines changed: 85 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,70 +1,109 @@
11
#!/bin/bash
22
set -e
33

4-
# Generate Test Summary from Pytest JUnit XML Output
5-
# Usage: ./generate-test-summary-pytest.sh <path-to-junit-xml>
4+
# Generate Detailed Test Summary from Multiple Pytest JUnit XML Output Files
5+
# Shows breakdown by test type (unit vs integration)
6+
# Usage: ./generate-test-summary-pytest-detailed.sh <unit-xml> <integration-xml>
67

7-
XML_FILE="${1:-test-results.xml}"
8+
UNIT_XML="${1:-}"
9+
INTEGRATION_XML="${2:-}"
810

911
echo "## Test Results" >> $GITHUB_STEP_SUMMARY
1012
echo "" >> $GITHUB_STEP_SUMMARY
1113

12-
# Parse test results from JUnit XML
13-
if [ -f "$XML_FILE" ]; then
14-
# Extract test counts from XML
15-
# JUnit XML structure: <testsuite tests="N" failures="N" errors="N" skipped="N">
14+
# Function to parse XML file
15+
parse_xml() {
16+
local xml_file="$1"
17+
local test_type="$2"
1618

17-
tests=$(grep -oP 'tests="\K[0-9]+' "$XML_FILE" | head -1)
18-
failures=$(grep -oP 'failures="\K[0-9]+' "$XML_FILE" | head -1)
19-
errors=$(grep -oP 'errors="\K[0-9]+' "$XML_FILE" | head -1)
20-
skipped=$(grep -oP 'skipped="\K[0-9]+' "$XML_FILE" | head -1)
19+
if [ ! -f "$xml_file" ]; then
20+
echo "0 0 0 0 0"
21+
return
22+
fi
23+
24+
tests=$(grep -oP 'tests="\K[0-9]+' "$xml_file" | head -1)
25+
failures=$(grep -oP 'failures="\K[0-9]+' "$xml_file" | head -1)
26+
errors=$(grep -oP 'errors="\K[0-9]+' "$xml_file" | head -1)
27+
skipped=$(grep -oP 'skipped="\K[0-9]+' "$xml_file" | head -1)
2128

22-
# Default to 0 if values are empty
2329
tests=${tests:-0}
2430
failures=${failures:-0}
2531
errors=${errors:-0}
2632
skipped=${skipped:-0}
27-
2833
passed=$((tests - failures - errors - skipped))
2934

30-
echo "| Status | Count |" >> $GITHUB_STEP_SUMMARY
31-
echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
32-
echo "| ✅ Passed | $passed |" >> $GITHUB_STEP_SUMMARY
33-
echo "| ❌ Failed | $((failures + errors)) |" >> $GITHUB_STEP_SUMMARY
34-
echo "| ⏭️ Skipped | $skipped |" >> $GITHUB_STEP_SUMMARY
35-
echo "| **Total** | **$tests** |" >> $GITHUB_STEP_SUMMARY
35+
echo "$tests $failures $errors $skipped $passed"
36+
}
37+
38+
# Parse both files
39+
read -r unit_tests unit_failures unit_errors unit_skipped unit_passed <<< "$(parse_xml "$UNIT_XML" "Unit")"
40+
read -r int_tests int_failures int_errors int_skipped int_passed <<< "$(parse_xml "$INTEGRATION_XML" "Integration")"
41+
42+
# Calculate totals
43+
total_tests=$((unit_tests + int_tests))
44+
total_failures=$((unit_failures + int_failures))
45+
total_errors=$((unit_errors + int_errors))
46+
total_skipped=$((unit_skipped + int_skipped))
47+
total_passed=$((unit_passed + int_passed))
48+
total_failed=$((total_failures + total_errors))
49+
50+
# Display detailed breakdown
51+
echo "### Summary by Test Type" >> $GITHUB_STEP_SUMMARY
52+
echo "" >> $GITHUB_STEP_SUMMARY
53+
echo "| Test Type | Passed | Failed | Skipped | Total |" >> $GITHUB_STEP_SUMMARY
54+
echo "|-----------|--------|--------|---------|-------|" >> $GITHUB_STEP_SUMMARY
55+
56+
if [ -f "$UNIT_XML" ]; then
57+
echo "| 🔧 Unit Tests | $unit_passed | $((unit_failures + unit_errors)) | $unit_skipped | $unit_tests |" >> $GITHUB_STEP_SUMMARY
58+
fi
59+
60+
if [ -f "$INTEGRATION_XML" ]; then
61+
echo "| 🔗 Integration Tests | $int_passed | $((int_failures + int_errors)) | $int_skipped | $int_tests |" >> $GITHUB_STEP_SUMMARY
62+
fi
63+
64+
echo "| **Total** | **$total_passed** | **$total_failed** | **$total_skipped** | **$total_tests** |" >> $GITHUB_STEP_SUMMARY
65+
echo "" >> $GITHUB_STEP_SUMMARY
66+
67+
# Overall status
68+
echo "### Overall Status" >> $GITHUB_STEP_SUMMARY
69+
echo "" >> $GITHUB_STEP_SUMMARY
70+
echo "| Status | Count |" >> $GITHUB_STEP_SUMMARY
71+
echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
72+
echo "| ✅ Passed | $total_passed |" >> $GITHUB_STEP_SUMMARY
73+
echo "| ❌ Failed | $total_failed |" >> $GITHUB_STEP_SUMMARY
74+
echo "| ⏭️ Skipped | $total_skipped |" >> $GITHUB_STEP_SUMMARY
75+
echo "| **Total** | **$total_tests** |" >> $GITHUB_STEP_SUMMARY
76+
echo "" >> $GITHUB_STEP_SUMMARY
77+
78+
# List failed tests if any
79+
if [ $total_failed -gt 0 ]; then
80+
echo "### ❌ Failed Tests" >> $GITHUB_STEP_SUMMARY
3681
echo "" >> $GITHUB_STEP_SUMMARY
3782

38-
# List failed tests if any
39-
if [ $((failures + errors)) -gt 0 ]; then
40-
echo "### ❌ Failed Tests" >> $GITHUB_STEP_SUMMARY
41-
echo "" >> $GITHUB_STEP_SUMMARY
42-
43-
# Extract failed test names from XML
44-
failed_tests_file=$(mktemp)
45-
46-
# Find testcase elements with failure or error children
47-
grep -oP '<testcase[^>]*classname="[^"]*"[^>]*name="[^"]*"[^>]*>.*?<(failure|error)' "$XML_FILE" | \
48-
grep -oP 'classname="\K[^"]*|name="\K[^"]*' | \
49-
paste -d '.' - - >> "$failed_tests_file" 2>/dev/null || true
50-
51-
if [ -s "$failed_tests_file" ]; then
52-
while IFS= read -r test; do
53-
echo "- \`$test\`" >> $GITHUB_STEP_SUMMARY
54-
done < "$failed_tests_file"
55-
else
56-
echo "_Unable to parse individual test names_" >> $GITHUB_STEP_SUMMARY
83+
failed_tests_file=$(mktemp)
84+
85+
# Extract failed tests from both files
86+
for xml_file in "$UNIT_XML" "$INTEGRATION_XML"; do
87+
if [ -f "$xml_file" ]; then
88+
grep -oP '<testcase[^>]*classname="[^"]*"[^>]*name="[^"]*"[^>]*>.*?<(failure|error)' "$xml_file" | \
89+
grep -oP 'classname="\K[^"]*|name="\K[^"]*' | \
90+
paste -d '.' - - >> "$failed_tests_file" 2>/dev/null || true
5791
fi
58-
59-
echo "" >> $GITHUB_STEP_SUMMARY
60-
echo "❌ **Tests failed!**" >> $GITHUB_STEP_SUMMARY
61-
rm -f "$failed_tests_file"
62-
exit 1
92+
done
93+
94+
if [ -s "$failed_tests_file" ]; then
95+
while IFS= read -r test; do
96+
echo "- \`$test\`" >> $GITHUB_STEP_SUMMARY
97+
done < "$failed_tests_file"
6398
else
64-
echo "✅ **All tests passed!**" >> $GITHUB_STEP_SUMMARY
99+
echo "_Unable to parse individual test names_" >> $GITHUB_STEP_SUMMARY
65100
fi
66-
else
67-
echo "⚠️ No test results found at: $XML_FILE" >> $GITHUB_STEP_SUMMARY
101+
102+
echo "" >> $GITHUB_STEP_SUMMARY
103+
echo "❌ **Tests failed!**" >> $GITHUB_STEP_SUMMARY
104+
rm -f "$failed_tests_file"
68105
exit 1
106+
else
107+
echo "✅ **All tests passed!**" >> $GITHUB_STEP_SUMMARY
69108
fi
70109

‎.github/workflows/run-python-tests.yml‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,14 @@ jobs:
3636
python -m pip install --upgrade pip
3737
pip install -r requirements.txt
3838
39-
- name: Run tests
40-
run: pytest --verbose --tb=short --junit-xml=test-results.xml || true
39+
- name: Run unit tests
40+
run: pytest -m unit --verbose --tb=short --junit-xml=test-results-unit.xml
41+
env:
42+
MONGO_URI: ${{ secrets.MFLIX_URI }}
43+
MONGO_DB: sample_mflix
44+
45+
- name: Run integration tests
46+
run: pytest -m integration --verbose --tb=short --junit-xml=test-results-integration.xml || true
4147
env:
4248
MONGO_URI: ${{ secrets.MFLIX_URI }}
4349
MONGO_DB: sample_mflix
@@ -48,7 +54,8 @@ jobs:
4854
with:
4955
name: test-results
5056
path: |
51-
server/python/test-results.xml
57+
server/python/test-results-unit.xml
58+
server/python/test-results-integration.xml
5259
server/python/htmlcov/
5360
retention-days: 30
5461

@@ -57,5 +64,6 @@ jobs:
5764
working-directory: .
5865
run: |
5966
chmod +x .github/scripts/generate-test-summary-pytest.sh
60-
.github/scripts/generate-test-summary-pytest.sh server/python/test-results.xml
61-
67+
.github/scripts/generate-test-summary-pytest.sh \
68+
server/python/test-results-unit.xml \
69+
server/python/test-results-integration.xml

‎server/python/tests/README.md‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
# Testing Guide for FastAPI MongoDB Sample Application
2+
3+
This document describes the testing strategy and how to run tests for the FastAPI MongoDB MFlix sample application.
4+
5+
## Test Structure
6+
7+
The test suite is organized into three categories:
8+
9+
### 1. **Schema Tests** (`test_movie_schemas.py`)
10+
- Tests Pydantic model validation
11+
- Validates request/response data structures
12+
- No database or external dependencies required
13+
- **10 tests** covering `CreateMovieRequest`, `UpdateMovieRequest`, and `Movie` models
14+
15+
### 2. **Unit Tests** (`test_movie_routes.py`)
16+
- Tests route handler functions in isolation
17+
- Uses `unittest.mock.AsyncMock` to mock MongoDB operations
18+
- No database connection required
19+
- Fast execution (< 2 seconds)
20+
- **51 tests** covering:
21+
- CRUD operations (create, read, update, delete)
22+
- Batch operations
23+
- Search functionality
24+
- Vector search
25+
- Aggregation pipelines
26+
27+
### 3. **Integration Tests** (`tests/integration/test_movie_routes_integration.py`)
28+
- Tests the full HTTP request/response cycle
29+
- Requires a running MongoDB instance with MFlix dataset
30+
- Uses a real server running in a subprocess
31+
- Tests are idempotent (clean up after themselves)
32+
- **8 tests** (7 passing, 1 skipped due to known bug) covering:
33+
- End-to-end CRUD operations
34+
- Search against real data
35+
- Batch operations with cleanup
36+
- Aggregation queries
37+
38+
## Running Tests
39+
40+
### Prerequisites
41+
42+
1. **For all tests:**
43+
```bash
44+
cd server/python
45+
source .venv/bin/activate # or `.venv\Scripts\activate` on Windows
46+
```
47+
48+
2. **For integration tests only:**
49+
- MongoDB instance running with MFlix dataset loaded
50+
- Connection string configured in `.env` file
51+
- Port 8001 available (used for test server)
52+
53+
### Run All Tests
54+
55+
```bash
56+
pytest tests/ -v
57+
```
58+
59+
**Expected output:** 68 passed, 1 skipped in ~5 seconds
60+
61+
### Run Only Unit Tests (Fast, No Database Required)
62+
63+
```bash
64+
pytest -m unit -v
65+
```
66+
67+
**Expected output:** 51 passed in ~1 second
68+
69+
### Run Only Integration Tests (Requires Database)
70+
71+
```bash
72+
pytest -m integration -v
73+
```
74+
75+
**Expected output:** 7 passed, 1 skipped in ~4 seconds
76+
77+
### Run Specific Test File
78+
79+
```bash
80+
# Schema tests
81+
pytest tests/test_movie_schemas.py -v
82+
83+
# Unit tests
84+
pytest tests/test_movie_routes.py -v
85+
86+
# Integration tests
87+
pytest tests/integration/test_movie_routes_integration.py -v
88+
```
89+
90+
### Run Specific Test Class or Method
91+
92+
```bash
93+
# Run a specific test class
94+
pytest tests/test_movie_routes.py::TestCreateMovie -v
95+
96+
# Run a specific test method
97+
pytest tests/test_movie_routes.py::TestCreateMovie::test_create_movie_success -v
98+
```
99+
100+
## Test Markers
101+
102+
Tests are marked with pytest markers for selective execution:
103+
104+
- `@pytest.mark.unit` - Unit tests with mocked dependencies
105+
- `@pytest.mark.integration` - Integration tests requiring database
106+
107+
## Integration Test Strategy
108+
109+
### Why Use a Running Server?
110+
111+
The integration tests start a real FastAPI server in a subprocess because:
112+
113+
1. **Event Loop Isolation**: AsyncMongoClient binds to the event loop it was created in. Using a real server avoids event loop conflicts.
114+
2. **Real-World Testing**: Tests the actual deployment configuration, including middleware, CORS, and startup events.
115+
3. **Educational Value**: Demonstrates a practical integration testing pattern for async Python applications.
116+
117+
### Idempotent Tests
118+
119+
All integration tests are designed to be idempotent:
120+
121+
- **Create operations**: Tests create new documents with unique identifiers
122+
- **Cleanup**: Fixtures automatically delete created documents after tests
123+
- **Read-only tests**: Tests against existing MFlix data don't modify anything
124+
- **Batch operations**: Create and delete multiple documents with proper cleanup
125+
126+
### Fixtures
127+
128+
Integration tests use pytest fixtures for test data lifecycle management:
129+
130+
- `client`: AsyncClient connected to the test server
131+
- `test_movie_data`: Sample movie data for creating test documents
132+
- `created_movie`: Creates a movie and cleans it up automatically
133+
- `multiple_test_movies`: Creates 3 movies for batch operation testing
134+
135+
## Known Issues
136+
137+
### Batch Create Bug (Skipped Test)
138+
139+
The `test_batch_create_movies` test is currently skipped due to a known bug in the API:
140+
141+
- **Issue**: `create_movies_batch` function calls `insert_many` twice (lines 1006 and 1015 in `movies.py`)
142+
- **Impact**: Causes 500 error on batch create operations
143+
- **Status**: To be fixed in a separate PR
144+
- **Test behavior**: Test detects the error and skips gracefully
145+
146+
## Troubleshooting
147+
148+
### Integration Tests Fail to Start Server
149+
150+
**Error**: `Port 8001 is already in use`
151+
152+
**Solution**:
153+
- Kill any process using port 8001: `lsof -ti:8001 | xargs kill -9`
154+
- Or change the port in `tests/integration/conftest.py`
155+
156+
### Integration Tests Can't Connect to MongoDB
157+
158+
**Error**: Connection timeout or authentication error
159+
160+
**Solution**:
161+
- Verify MongoDB is running
162+
- Check `.env` file has correct `MONGODB_URI`
163+
- Ensure MFlix dataset is loaded
164+
- Test connection: `mongosh <your-connection-string>`
165+
166+
### Unit Tests Fail with Import Errors
167+
168+
**Error**: `ModuleNotFoundError`
169+
170+
**Solution**:
171+
- Ensure virtual environment is activated
172+
- Install dependencies: `pip install -r requirements.txt`
173+
- Run from `server/python` directory
174+
175+
## Contributing
176+
177+
When adding new routes or functionality:
178+
179+
1. **Add unit tests** in `test_movie_routes.py` with mocked dependencies
180+
2. **Add integration tests** in `tests/integration/test_movie_routes_integration.py` for end-to-end validation
181+
3. **Use appropriate markers** (`@pytest.mark.unit` or `@pytest.mark.integration`)
182+
4. **Follow fixture patterns** for test data lifecycle management
183+
5. **Ensure idempotency** - tests should clean up after themselves
184+
6. **Document test purpose** with clear docstrings
185+
186+
## Additional Resources
187+
188+
- [pytest documentation](https://docs.pytest.org/)
189+
- [pytest-asyncio documentation](https://pytest-asyncio.readthedocs.io/)
190+
- [FastAPI testing guide](https://fastapi.tiangolo.com/tutorial/testing/)
191+
- [MongoDB Motor documentation](https://motor.readthedocs.io/)
192+

‎server/python/tests/integration/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
"""Integration tests for the FastAPI MongoDB sample application."""
2+

0 commit comments

Comments
 (0)