diff --git a/.github/actions/pytest/action.yml b/.github/actions/pytest/action.yml index 0037129a5e..af838d4af3 100644 --- a/.github/actions/pytest/action.yml +++ b/.github/actions/pytest/action.yml @@ -73,6 +73,11 @@ runs: shell: bash run: | + # Sanitize test_type for filenames (always set this for artifact upload) + # Remove commas and spaces from test_type for use in filenames + STR_TEST_TYPE=$(echo "${{ inputs.test_type }}" | tr ', ' '_') + echo "STR_TEST_TYPE=${STR_TEST_TYPE}" >> $GITHUB_ENV + # Check for JUnit XML file and determine test status JUNIT_FILE="test-results/pytest_test_report.xml" @@ -84,17 +89,24 @@ runs: ERROR_TESTS=$(grep -o 'errors="[0-9]*"' "$JUNIT_FILE" | grep -o '[0-9]*' | head -1 || echo "0") echo "๐Ÿ“Š ${TOTAL_TESTS} tests completed (${FAILED_TESTS} failed, ${ERROR_TESTS} errors)" - # Create metadata file with step context information - METADATA_FILE="test-results/test_metadata.json" + # Create uniquely named metadata file with step context information + # Use framework-testtype-arch to make it unique per test run + METADATA_FILE="test-results/test_metadata_${{ inputs.framework }}_${STR_TEST_TYPE}_${{ inputs.platform_arch }}.json" + JUNIT_NAME="pytest_test_report_${{ inputs.framework }}_${STR_TEST_TYPE}_${{ inputs.platform_arch }}.xml" + + # Rename XML file to unique name + mv "$JUNIT_FILE" "test-results/$JUNIT_NAME" + echo '{' > "$METADATA_FILE" echo ' "job_name": "${{ github.job }}",' >> "$METADATA_FILE" echo ' "framework": "${{ inputs.framework }}",' >> "$METADATA_FILE" echo ' "test_type": "${{ inputs.test_type }}",' >> "$METADATA_FILE" echo ' "platform_arch": "${{ inputs.platform_arch }}",' >> "$METADATA_FILE" - echo ' "junit_xml_file": "pytest_test_report.xml",' >> "$METADATA_FILE" + echo ' "junit_xml_file": "'"$JUNIT_NAME"'",' >> "$METADATA_FILE" echo ' "step_name": "Run ${{ inputs.test_type }} tests"' >> "$METADATA_FILE" echo '}' >> "$METADATA_FILE" - echo "๐Ÿ“ Created test metadata file" + echo "๐Ÿ“ Created test metadata file: $METADATA_FILE" + echo "๐Ÿ“ Renamed XML file to: $JUNIT_NAME" else echo "โš ๏ธ JUnit XML file not found - test results may not be available for upload" TOTAL_TESTS=0 @@ -109,6 +121,8 @@ runs: uses: actions/upload-artifact@v4 if: always() # Always upload test results, even if tests failed with: - name: test-results-${{ inputs.framework }}-${{ inputs.test_type }}-${{ env.PLATFORM_ARCH }} - path: test-results/${{ env.PYTEST_XML_FILE }} + name: test-results-${{ inputs.framework }}-${{ env.STR_TEST_TYPE }}-${{ env.PLATFORM_ARCH }} + path: | + test-results/pytest_test_report_${{ inputs.framework }}_${{ env.STR_TEST_TYPE }}_${{ inputs.platform_arch }}.xml + test-results/test_metadata_${{ inputs.framework }}_${{ env.STR_TEST_TYPE }}_${{ inputs.platform_arch }}.json retention-days: 7 \ No newline at end of file diff --git a/.github/workflows/upload_complete_workflow_metrics.py b/.github/workflows/upload_complete_workflow_metrics.py index 1c0cc57fc5..99d110bfa1 100644 --- a/.github/workflows/upload_complete_workflow_metrics.py +++ b/.github/workflows/upload_complete_workflow_metrics.py @@ -825,8 +825,45 @@ def _upload_test_metrics(self, job_data: Dict[str, Any]) -> None: job_name = job_data.get("name", "") job_id = str(job_data["id"]) + # Skip deployment test jobs (No pytest metadata files are created) + if job_name.lower().startswith("deploy"): + print(f"โญ๏ธ Skipping test metrics for deployment job '{job_name}'") + return + print(f"๐Ÿงช Looking for test results for job '{job_name}'") + # Determine framework from job name to filter metadata files + framework = None + job_name_lower = job_name.lower() + if "vllm" in job_name_lower: + framework = "vllm" + elif "sglang" in job_name_lower: + framework = "sglang" + elif "trtllm" in job_name_lower: + framework = "trtllm" + + if not framework: + print(f"โš ๏ธ Could not determine framework from job name: {job_name}") + return + + # Determine platform architecture from job name + # Job names typically look like: "vllm (amd64)" or "sglang (arm64)" + platform_arch = None + if "(amd64)" in job_name_lower or "amd64" in job_name_lower: + platform_arch = "amd64" + elif "(arm64)" in job_name_lower or "arm64" in job_name_lower: + platform_arch = "arm64" + + if not platform_arch: + print( + f"โš ๏ธ Could not determine platform architecture from job name: {job_name}" + ) + # Default to amd64 if not specified + platform_arch = "amd64" + print(f" Defaulting to platform_arch: {platform_arch}") + + print(f"๐Ÿ“ฆ Job framework: {framework}, platform_arch: {platform_arch}") + # Look for test results directory test_results_dir = "test-results" if not os.path.exists(test_results_dir): @@ -834,13 +871,21 @@ def _upload_test_metrics(self, job_data: Dict[str, Any]) -> None: return # Look for metadata files to get accurate step and framework info - metadata_files = glob.glob(f"{test_results_dir}/test_metadata.json") + # Updated pattern to match new unique naming: test_metadata___.json + # Filter by both framework AND architecture to only process this job's tests + metadata_files = glob.glob( + f"{test_results_dir}/test_metadata_{framework}_*_{platform_arch}.json" + ) if not metadata_files: - print(f"โš ๏ธ No test metadata files found in {test_results_dir}") + print( + f"โš ๏ธ No test metadata files found for framework '{framework}' with arch '{platform_arch}' in {test_results_dir}" + ) return - print(f"๐Ÿ“„ Found {len(metadata_files)} test metadata files") + print( + f"๐Ÿ“„ Found {len(metadata_files)} test metadata files for {framework} ({platform_arch})" + ) total_tests_processed = 0