diff --git a/tools/testing/README.md b/tools/testing/README.md new file mode 100644 index 0000000000..04a45d0c75 --- /dev/null +++ b/tools/testing/README.md @@ -0,0 +1,205 @@ +# Scopy Testing Tools + +A comprehensive set of Python tools for managing Scopy's manual testing workflow. These tools automate test environment setup, result parsing, and CSV template generation for release testing processes. + +## Quick Start + +### 1. **Generate Test Environment** +```bash +# Create test environment for v3.0.0 with all tests +python3 setup_test_environment.py v3.0.0 + +# Filter by component +python3 setup_test_environment.py v3.0.0 --component adc dac + +# Filter by priority level +python3 setup_test_environment.py v3.0.0 --rbp P0 P1 + +# Release testing: P0 tests + new features since last version +python3 setup_test_environment.py v3.0.0 --rbp P0 --new-since v2.5.0 +``` +### 2. **CSV Template Auto-Generated** +The CSV template is automatically created during step 1. You can also generate it manually: +```bash +python3 parseTestResults.py v3.0.0 +``` + +### 3. **Execute Tests Manually** +Edit the generated csv file to add test results based on provided columns + + +## Detailed Usage + +### `setup_test_environment.py` - Test Environment Setup + +**Purpose:** Creates filtered copies of test documentation for manual testing execution. + +**Basic Usage:** +```bash +python3 setup_test_environment.py [OPTIONS] +``` + +**Parameters:** +- `version` - Version name for the test directory (e.g., v3.0.0) +- `--rbp` - Filter by RBP priority levels (P0, P1, P2, P3) +- `--component` - Filter by component names (adc, dac, m2k, core, etc.) +- `--new-since` - Include tests added since specified git version (overrides all other filters so that all new tests are added) + +**Available Components:** +Components are determined by folder structure in `docs/tests/`: +- **Plugin components:** Any subdirectory in `docs/tests/plugins/` (e.g., adc, dac, m2k, ad936x, ad9084, adrv9002, fmcomms5, swiot1l, datalogger, debugger, jesd, pqm, registermap) +- **Core component:** Files in `docs/tests/general/core/` subdirectory +- **General meta-component:** Using `--component general` includes ALL files from the `general/` directory, including both core files and other general test files + +**Filter Logic:** +- When using `--component` and/or `--rbp` without `--new-since`: Tests must match ALL specified filters (AND logic) +- When using `--new-since`: Filter logic becomes `(component AND rbp) OR new-since` + - This means all new tests are included regardless of component/rbp filters + - Plus any existing tests matching the component/rbp criteria + +**Examples:** +```bash +# Copy all tests for v3.0.0 +python3 setup_test_environment.py v3.0.0 + +# Only P0 and P1 priority tests +python3 setup_test_environment.py v3.0.0 --rbp P0 P1 + +# Only ADC and DAC component tests +python3 setup_test_environment.py v3.0.0 --component adc dac + +# Core component tests only +python3 setup_test_environment.py v3.0.0 --component core + +# All general tests (includes core and all other general/ files) +python3 setup_test_environment.py v3.0.0 --component general + +# Combined filters: M2K component with P0 priority +python3 setup_test_environment.py v3.0.0 --component m2k --rbp P0 + +# Release testing: All P0 tests + All new tests since v2.5.0 +python3 setup_test_environment.py v3.0.0 --rbp P0 --new-since v2.5.0 + +# Only new features testing +python3 setup_test_environment.py v3.0.0 --new-since v2.5.0 +``` + +**Output:** +- Creates `testing_results/testing_results_v3.0.0/` directory +- Contains filtered RST files ready for manual testing +- Automatically generates CSV template +- If directory exists, prompts to overwrite (this will also delete and regenerate the CSV file) + +### `parseTestResults.py` - CSV Template Generation + +**Purpose:** Generates CSV templates from testing directories for manual test execution tracking. + +**Basic Usage:** +```bash +python3 parseTestResults.py [OPTIONS] +``` + +**Parameters:** +- `version` - Version to parse (must match testing_results directory) +- `--output` - Custom output CSV file path (optional) +- `--force` - Force overwrite without prompting (optional) + +**Examples:** +```bash +# Parse v3.0.0 test results +python3 parseTestResults.py v3.0.0 + +# Custom output location +python3 parseTestResults.py v3.0.0 --output /path/to/custom_results.csv + +# Force overwrite without prompting +python3 parseTestResults.py v3.0.0 --force +``` + +**Output:** +- Creates `testing_results/testing_results_{version}.csv` +- Template with empty result fields for manual completion +- Columns: Test UID | Component | RBP | Result | Tested OS | Comments | Tester | File + +## Complete Testing Workflow + +### Release Testing Example (v3.0.0) + +**1. Prepare Test Environment** +```bash +# Create comprehensive release testing environment +python3 setup_test_environment.py v3.0.0 --rbp P0 --new-since v2.5.0 +``` + +**2. Execute Tests and Track Results** +- Open the auto-generated CSV template: `testing_results/testing_results_v3.0.0.csv` +- For each test, fill in: + - **Result:** PASS, FAIL, or SKIP + - **Tested OS:** Operating system used + - **Comments:** Any observations or notes + - **Tester:** Your name or identifier + +**3. Review and Analyze** +- Save the completed CSV file +- Use spreadsheet software for analysis and reporting +- Historical data is preserved across versions + + + +## Requirements + +### Git Requirements +- Git repository (required for `--new-since` functionality) +- Proper git tags for version references + +### File Structure Requirements +``` +scopy/ +├── docs/tests/ # Source test documentation +├── testing_results/ # Generated test environments +└── tools/testing/ # This toolset +``` + +## CSV Template Format + +### Columns +- **Test UID** - Unique test identifier +- **Component** - Test component (adc, dac, m2k, core, etc.) +- **RBP** - Risk-based priority (P0, P1, P2, P3) +- **Result** - Test outcome (empty template field for PASSED/FAILED/SKIPPED) +- **Tested OS** - Operating system used for testing (empty template field) +- **Comments** - Test execution notes (empty template field) +- **Tester** - Person executing the tests (empty template field) +- **File** - Source RST filename + +### Multiple Versions +- Each version gets its own CSV file: `testing_results_{version}.csv` +- Historical test data preserved across versions +- Easy import into spreadsheet tools for analysis + +## Troubleshooting + +### Common Issues + +**"Git tag not found" error:** +```bash +# Check available git tags +git tag -l + +# Use exact tag name +python3 setup_test_environment.py v3.0.0 --new-since v2.5.0 +``` + +**"Not in a git repository" error:** +```bash +# Ensure you're in the scopy git repository root +cd /path/to/scopy +python3 tools/testing/setup_test_environment.py v3.0.0 --new-since v2.5.0 +``` + +**No tests found:** +- Verify `docs/tests/` directory exists +- Check test files contain proper `**UID:**` and `**RBP:**` metadata + + + diff --git a/tools/testing/file_filters.py b/tools/testing/file_filters.py new file mode 100644 index 0000000000..ffec7ffbb2 --- /dev/null +++ b/tools/testing/file_filters.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +""" +File Filters Module +Handles file and content filtering logic for test environment setup. + +This module provides functionality for: +- Component-based file filtering +- RBP-based content filtering +- New test detection and filtering +- Helper functions for filtering decisions +""" + +from rst_parser import parse_rst_structure, rebuild_rst_file + + +def _should_include_file(test_file, component_filter, new_uids): + """Determine if file should be included based on filters.""" + # Skip all index files - they're not needed for CSV workflow + if 'index.rst' in test_file['relative_path']: + return False + + # Special handling: 'general' meta-component includes entire general/ directory + # This includes both 'core' subdirectory and 'general' files + if 'general' in component_filter: + if test_file['relative_path'].startswith('general/'): + return True + + # Include files from specified components + if test_file['component'] in component_filter: + return True + + # Include files containing new tests (bypass component filter) + if new_uids and any(test['uid'] in new_uids for test in test_file['tests']): + return True + + return False + + +def filter_by_component(test_files, component_filter, new_uids=None): + """Simple file-level filtering by component, with new-since support.""" + if not component_filter: + return test_files + + return [tf for tf in test_files + if _should_include_file(tf, component_filter, new_uids)] + + +def _should_include_test(test, rbp_filter, new_uids): + """Determine if test should be included based on filters.""" + # Include if it's a new test (always) + if new_uids and test['uid'] in new_uids: + return True + + # Include if matches RBP filter + if rbp_filter and test['rbp'] in rbp_filter: + return True + + # Include if no filters specified + if not rbp_filter and not new_uids: + return True + + return False + + +def _extract_all_uids(test_files): + """Extract all UIDs from test files efficiently.""" + return {test['uid'] for file_info in test_files + for test in file_info['tests'] + if test['uid'] != 'MISSING'} + + +def filter_rst_content(file_path, rbp_filter, new_uids=None): + """Parse RST file and keep only tests matching RBP filter or new tests.""" + + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Split into header + test sections + structure = parse_rst_structure(content) + + # Filter tests by RBP or new-since + filtered_tests = [test for test in structure['tests'] + if _should_include_test(test, rbp_filter, new_uids)] + + # Reconstruct file + return rebuild_rst_file(structure['header'], filtered_tests) \ No newline at end of file diff --git a/tools/testing/file_operations.py b/tools/testing/file_operations.py new file mode 100644 index 0000000000..c2631e0353 --- /dev/null +++ b/tools/testing/file_operations.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +""" +File Operations Module +Handles file copying operations and index file management. + +This module provides functionality for: +- Copying test files with or without modification +- Managing index file toctree entries +- Component tracking for copied files +- RST content-level filtering and file operations +""" + +import os +import shutil +from file_filters import filter_rst_content + + +def copy_files_unchanged(test_files, dest_dir, component_filter=None): + """Copy files without content modification.""" + copied_count = 0 + + for test_file in test_files: + src_path = test_file['file_path'] + dest_path = os.path.join(dest_dir, test_file['relative_path']) + + # Create destination directory + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + + # Copy file unchanged + shutil.copy2(src_path, dest_path) + copied_count += 1 + + return copied_count + + +def filter_by_rbp_content(test_files, rbp_filter, dest_dir, component_filter=None, new_uids=None): + """Content-level filtering: modify files to keep only specified RBP tests or new tests.""" + if not rbp_filter and not new_uids: + # No filters - copy files as-is + return copy_files_unchanged(test_files, dest_dir, component_filter) + + copied_count = 0 + + # Filter active - parse and modify content + for test_file in test_files: + dest_path = os.path.join(dest_dir, test_file['relative_path']) + + # Create destination directory + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + + if test_file['tests']: # File contains tests + filtered_content = filter_rst_content(test_file['file_path'], rbp_filter, new_uids) + with open(dest_path, 'w', encoding='utf-8') as f: + f.write(filtered_content) + else: # File has no tests (docs, etc.) + shutil.copy2(test_file['file_path'], dest_path) + + copied_count += 1 + + return copied_count \ No newline at end of file diff --git a/tools/testing/git_operations.py b/tools/testing/git_operations.py new file mode 100644 index 0000000000..7981b296d5 --- /dev/null +++ b/tools/testing/git_operations.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +""" +Git Operations Module +Handles git repository operations and new test detection. + +This module provides functionality for: +- Git repository validation and operations +- Extracting test files from specific git versions +- Detecting new tests between versions +""" + +import os +import subprocess +import tempfile +import shutil +from rst_parser import scan_test_files + + +def get_git_root(): + """Get the git repository root directory.""" + try: + result = subprocess.run(['git', 'rev-parse', '--show-toplevel'], + capture_output=True, check=True, text=True) + return result.stdout.strip() + except subprocess.CalledProcessError: + return None + + +def validate_git_repository(): + """Ensure we're in a git repository.""" + return get_git_root() is not None + + +def git_tag_exists(tag_name): + """Check if a git tag exists.""" + git_root = get_git_root() + if not git_root: + return False + + try: + result = subprocess.run(['git', 'tag', '-l', tag_name], + cwd=git_root, capture_output=True, check=True, text=True) + return tag_name in result.stdout.strip().split('\n') + except subprocess.CalledProcessError: + return False + + +def detect_new_tests(baseline_version): + """Detect new test UIDs since baseline version.""" + git_root = get_git_root() + if not git_root: + raise RuntimeError("Not in a git repository") + + temp_dir = tempfile.mkdtemp() + + try: + # Extract docs/tests directory at baseline version from git root + cmd = ['git', 'archive', baseline_version, 'docs/tests'] + with subprocess.Popen(cmd, cwd=git_root, stdout=subprocess.PIPE) as git_proc: + subprocess.run(['tar', '-x', '-C', temp_dir], + stdin=git_proc.stdout, check=True) + + # Scan extracted files for UIDs + baseline_tests_dir = os.path.join(temp_dir, 'docs', 'tests') + if not os.path.exists(baseline_tests_dir): + raise FileNotFoundError(f"docs/tests not found in {baseline_version}") + + baseline_files = scan_test_files(baseline_tests_dir) + + baseline_uids = set() + for file_info in baseline_files: + for test in file_info['tests']: + if test['uid'] != 'MISSING': + baseline_uids.add(test['uid']) + + print(f"Baseline version {baseline_version}: {len(baseline_uids)} tests") + return baseline_uids + + except Exception as e: + print(f"Error extracting baseline UIDs: {e}") + raise + finally: + shutil.rmtree(temp_dir) \ No newline at end of file diff --git a/tools/testing/parseTestResults.py b/tools/testing/parseTestResults.py new file mode 100644 index 0000000000..3938b0ee11 --- /dev/null +++ b/tools/testing/parseTestResults.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +""" +Generate CSV templates from testing_results directories for manual test execution. +Creates version-specific CSV files with test metadata for tracking test execution. + +Usage: + python3 parseTestResults.py v3.0.0 # Generate v3.0.0 CSV template + python3 parseTestResults.py v3.0.0 --output custom.csv # Custom output file +""" + +import os +import sys +import re +import argparse +import csv +from pathlib import Path + + +def build_testing_results_path(version): + """Build testing results path from version string.""" + script_dir = os.path.dirname(os.path.abspath(__file__)) + scopy_root = os.path.dirname(os.path.dirname(script_dir)) # tools/testing/ → scopy/ + return os.path.join(scopy_root, "testing_results", f"testing_results_{version}") + + +def get_output_csv_path(version, custom_path=None): + """Get CSV output path (default: testing_results/testing_results_v{version}.csv).""" + if custom_path: + return custom_path + + script_dir = os.path.dirname(os.path.abspath(__file__)) + scopy_root = os.path.dirname(os.path.dirname(script_dir)) # tools/testing/ → scopy/ + return os.path.join(scopy_root, "testing_results", f"testing_results_{version}.csv") + + +def scan_testing_results_files(results_dir): + """Find all RST files in testing results directory.""" + rst_files = [] + + for root, dirs, files in os.walk(results_dir): + for file in files: + if file.endswith('.rst'): + file_path = os.path.join(root, file) + relative_path = os.path.relpath(file_path, results_dir) + component = extract_component_from_testing_path(relative_path) + + rst_files.append({ + 'file_path': file_path, + 'relative_path': relative_path, + 'component': component, + 'filename': file + }) + + return rst_files + + +def extract_component_from_testing_path(relative_path): + """Extract component from testing results file path.""" + parts = relative_path.split('/') + + if parts[0] == 'plugins' and len(parts) >= 2: + return parts[1] # plugins/adc/... → 'adc' + elif parts[0] == 'general': + if len(parts) >= 2 and parts[1] == 'core': + return 'core' + elif 'preferences' in parts[-1]: + return 'preferences' + elif 'package_manager' in parts[-1]: + return 'package_manager' + elif 'scripting' in parts[-1]: + return 'scripting' + elif 'instrument_detaching' in parts[-1]: + return 'instrument_detaching' + else: + return 'general' + else: + return 'other' + + +def parse_rst_structure(content): + """Parse RST into header section + individual test sections.""" + # Find test boundaries using pattern: "Test N " followed by underline + test_pattern = r'^(Test \d+[^\n]*\n[\^=-]+\n)' + + # Split content at test boundaries + parts = re.split(test_pattern, content, flags=re.MULTILINE) + + if len(parts) == 1: + # No tests found + return {'header': content, 'tests': []} + + header = parts[0] # Everything before first test + tests = [] + + # Process test sections + for i in range(1, len(parts), 2): + if i + 1 < len(parts): + heading = parts[i] + content_block = parts[i + 1] + + # Extract UID/RBP from content + uid = extract_uid_from_content(content_block) + rbp = extract_rbp_from_content(content_block) + + tests.append({ + 'heading': heading, + 'content': content_block, + 'uid': uid, + 'rbp': rbp + }) + + return {'header': header, 'tests': tests} + + +def extract_uid_from_content(content): + """Extract UID from test content block.""" + uid_pattern = r'\*\*UID:\*\*\s+([A-Z_][A-Z0-9_.]*)' + match = re.search(uid_pattern, content) + return match.group(1) if match else 'MISSING' + + +def extract_rbp_from_content(content): + """Extract RBP from test content block.""" + rbp_pattern = r'\*\*RBP:\*\*\s+(P[0-3])' + match = re.search(rbp_pattern, content) + return match.group(1) if match else 'MISSING' + + + + +def parse_test_metadata_from_rst(file_path): + """Parse RST file and extract test metadata (UID, RBP only).""" + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + structure = parse_rst_structure(content) + return structure['tests'] + + except Exception as e: + print(f"Warning: Error parsing {file_path}: {e}") + return [] + + +def process_all_test_data(results_dir): + """Main processing function - extract test metadata from directory.""" + all_test_data = [] + + rst_files = scan_testing_results_files(results_dir) + + for rst_file in rst_files: + # Skip index files - they don't contain tests + if 'index.rst' in rst_file['filename']: + continue + + tests = parse_test_metadata_from_rst(rst_file['file_path']) + + for test in tests: + test_data = { + 'uid': test['uid'], + 'component': rst_file['component'], + 'rbp': test['rbp'], + 'result': '', + 'tested_os': '', + 'comments': '', + 'tester': '', + 'file': rst_file['filename'] + } + all_test_data.append(test_data) + + return all_test_data + + +def validate_and_fill_missing_data(test_data): + """Ensure all fields have values (empty strings for missing).""" + for test in test_data: + for key, value in test.items(): + if value is None : + test[key] = '' + + return test_data + + + + +def generate_csv(test_data, csv_path, force=False): + """Generate CSV file with test data template.""" + + # Check if file exists and prompt user (unless force is True) + if os.path.exists(csv_path) and not force: + response = input(f"File {csv_path} exists. Overwrite? (y/N): ") + if response.lower() != 'y': + print("Aborted.") + return False + + + # CSV file header + headers = ['Test UID', 'Component', 'RBP', 'Result', 'Tested OS', 'Comments', 'Tester', 'File'] + + with open(csv_path, 'w', newline='', encoding='utf-8') as csvfile: + writer = csv.writer(csvfile) + + # Write header row + writer.writerow(headers) + + # Write test data rows + for test in test_data: + writer.writerow([ + test['uid'], + test['component'], + test['rbp'], + test['result'], + test['tested_os'], + test['comments'], + test['tester'], + test['file'] + ]) + + return True + +def main(): + parser = argparse.ArgumentParser( + description='Generate CSV template from Scopy test metadata', + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('version', help='Version to process (e.g., v3.0.0)') + parser.add_argument('--output', help='Custom output CSV file path') + parser.add_argument('--force', action='store_true', help='Force overwrite without prompting') + + args = parser.parse_args() + + # Build paths + results_dir = build_testing_results_path(args.version) + csv_path = get_output_csv_path(args.version, args.output) + + # Validate input directory exists + if not os.path.exists(results_dir): + print(f"Error: Testing results directory not found: {results_dir}") + print(f"Make sure you've run setup_test_environment.py for version {args.version}") + sys.exit(1) + + # Create output directory if needed + output_dir = os.path.dirname(csv_path) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + # Process all test data + test_data = process_all_test_data(results_dir) + test_data = validate_and_fill_missing_data(test_data) + + if not test_data: + print("Warning: No test data found in the specified directory.") + print("Make sure the directory contains RST files with test metadata.") + sys.exit(1) + + # Generate CSV report + success = generate_csv(test_data, csv_path, args.force) + + if not success: + sys.exit(1) + + print("CSV generated successfully") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/testing/rst_parser.py b/tools/testing/rst_parser.py new file mode 100644 index 0000000000..18254e1f23 --- /dev/null +++ b/tools/testing/rst_parser.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +""" +RST Parser Module +Handles parsing of reStructuredText test files and component mapping. + +This module provides the core functionality for: +- Extracting test metadata (UID, RBP) from RST files +- Parsing RST structure into header and test sections +- Component mapping from file paths +- RST file reconstruction +""" + +import os +import re + +# Regex patterns for UID/RBP extraction +UID_PATTERN = r'\*\*UID:\*\*\s+([A-Z_][A-Z0-9_.]*)' +RBP_PATTERN = r'\*\*RBP:\*\*\s+(P[0-3])' + + +def scan_test_files(source_dir): + """Find all RST files and extract test metadata.""" + test_files = [] + for root, dirs, files in os.walk(source_dir): + for file in files: + if file.endswith('.rst'): + file_path = os.path.join(root, file) + relative_path = os.path.relpath(file_path, source_dir) + component = extract_component_from_path(relative_path) + tests = parse_rst_tests(file_path) + + test_files.append({ + 'file_path': file_path, + 'relative_path': relative_path, + 'component': component, + 'tests': tests + }) + return test_files + + +def parse_rst_tests(file_path): + """Extract UID/RBP pairs from RST file.""" + tests = [] + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Find UID patterns using shared constant + uid_matches = re.findall(UID_PATTERN, content) + + for uid in uid_matches: + uid_pos = content.find(f"**UID:** {uid}") + if uid_pos == -1: + continue + + # Search for RBP in the next 500 characters after UID + search_area = content[uid_pos:uid_pos + 500] + rbp = extract_rbp_from_content(search_area) + + tests.append({'uid': uid, 'rbp': rbp}) + except Exception as e: + print(f"Warning: Error parsing {file_path}: {e}") + + return tests + + +def extract_component_from_path(relative_path): + """Map file path to component name.""" + parts = relative_path.split('/') + + if parts[0] == 'plugins' and len(parts) >= 2: + return parts[1] # plugins/adc/... -> 'adc' + elif parts[0] == 'general': + if len(parts) >= 2 and parts[1] == 'core': + return 'core' + else: + return 'general' + else: + return 'other' + + +def parse_rst_structure(content): + """Parse RST into header section + individual test sections.""" + + # Find test boundaries using pattern: "Test N " followed by underline + test_pattern = r'^(Test \d+[^\n]*\n[\^=-]+\n)' + + # Split content at test boundaries + parts = re.split(test_pattern, content, flags=re.MULTILINE) + + if len(parts) == 1: + # No tests found + return {'header': content, 'tests': []} + + header = parts[0] # Everything before first test + tests = [] + + # Process test sections (parts[1], parts[2], parts[3], parts[4], ...) + # parts[1] = "Test 1 " heading, parts[2] = test content + # parts[3] = "Test 2 " heading, parts[4] = test content, etc. + for i in range(1, len(parts), 2): + if i + 1 < len(parts): + heading = parts[i] + content_block = parts[i + 1] + + # Extract UID/RBP from content + uid = extract_uid_from_content(content_block) + rbp = extract_rbp_from_content(content_block) + + tests.append({ + 'heading': heading, + 'content': content_block, + 'uid': uid, + 'rbp': rbp + }) + + return {'header': header, 'tests': tests} + + +def extract_uid_from_content(content): + """Extract UID from test content block.""" + match = re.search(UID_PATTERN, content) + return match.group(1) if match else 'MISSING' + + +def extract_rbp_from_content(content): + """Extract RBP from test content block.""" + match = re.search(RBP_PATTERN, content) + return match.group(1) if match else 'MISSING' + + +def rebuild_rst_file(header, filtered_tests): + """Reconstruct RST file with only filtered tests.""" + result = header + + for test in filtered_tests: + result += test['heading'] # "Test N:" heading with underline + result += test['content'] # Test content + + return result \ No newline at end of file diff --git a/tools/testing/setup_test_environment.py b/tools/testing/setup_test_environment.py new file mode 100755 index 0000000000..17fa8ca23a --- /dev/null +++ b/tools/testing/setup_test_environment.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +""" +Test Environment Setup Script +Creates filtered copies of test documentation for manual testing execution. + +Usage: + python3 setup_test_environment.py v3.0.0 # Copy all tests + python3 setup_test_environment.py v3.0.0 --rbp P0 P1 # Copy P0/P1 tests only + python3 setup_test_environment.py v3.0.0 --component adc dac # Copy adc/dac tests only + python3 setup_test_environment.py v3.0.0 --rbp P0 --component m2k # Combined filtering + python3 setup_test_environment.py v3.0.0 --new-since v2.1.0 # New tests since v2.1.0 + python3 setup_test_environment.py v3.0.0 --rbp P0 --new-since v2.1.0 # P0 + new tests +""" + +import os +import sys +import shutil +import argparse + +# Import from our new modules +from git_operations import validate_git_repository, git_tag_exists, detect_new_tests +from rst_parser import scan_test_files +from file_filters import filter_by_component, _extract_all_uids +from file_operations import copy_files_unchanged, filter_by_rbp_content + + +def main(): + parser = argparse.ArgumentParser( + description='Setup test environment for manual testing', + epilog='Examples:\n' + ' %(prog)s v3.0.0\n' + ' %(prog)s v3.0.0 --rbp P0 P1\n' + ' %(prog)s v3.0.0 --component adc m2k\n' + ' %(prog)s v3.0.0 --rbp P0 --component m2k', + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('version', help='Version name (e.g., v3.0.0)') + parser.add_argument('--rbp', nargs='+', choices=['P0', 'P1', 'P2', 'P3'], + help='Filter by RBP priority levels') + parser.add_argument('--component', nargs='+', + help='Filter by component names (adc, m2k, core, etc.)') + parser.add_argument('--new-since', + help='Include tests added since specified version (e.g., v2.1.0)') + + args = parser.parse_args() + + # Validate git requirements for --new-since + if args.new_since: + if not validate_git_repository(): + print("Error: Not in a git repository. --new-since requires git.") + sys.exit(1) + if not git_tag_exists(args.new_since): + print(f"Error: Git tag '{args.new_since}' not found.") + sys.exit(1) + + # Setup paths + script_dir = os.path.dirname(os.path.abspath(__file__)) + scopy_root = os.path.dirname(os.path.dirname(script_dir)) # tools/testing -> root + + # Validate component names + if args.component: + valid_components = [] + + # Check plugins directory + plugins_dir = os.path.join(scopy_root, 'docs', 'tests', 'plugins') + if os.path.exists(plugins_dir): + valid_components.extend([d for d in os.listdir(plugins_dir) + if os.path.isdir(os.path.join(plugins_dir, d))]) + + # Check general directory + general_dir = os.path.join(scopy_root, 'docs', 'tests', 'general') + if os.path.exists(general_dir): + general_components = [d for d in os.listdir(general_dir) + if os.path.isdir(os.path.join(general_dir, d))] + valid_components.extend(general_components) + valid_components.append('general') # 'general' itself is also valid + + for component in args.component: + if component not in valid_components: + print(f"Error: Component '{component}' not found.") + sys.exit(1) + + # Continue with paths + source_dir = os.path.join(scopy_root, 'docs', 'tests') + testing_results_base = os.path.join(scopy_root, 'testing_results') + dest_dir = os.path.join(testing_results_base, f'testing_results_{args.version}') + + # Validate source directory + if not os.path.exists(source_dir): + print(f"Error: Source directory not found: {source_dir}") + sys.exit(1) + + # Create base testing_results directory if it doesn't exist + if not os.path.exists(testing_results_base): + os.makedirs(testing_results_base) + + # Check for existing directory and CSV file + csv_path = os.path.join(testing_results_base, f'testing_results_{args.version}.csv') + + # Create destination directory + if os.path.exists(dest_dir): + response = input(f"Directory {dest_dir} exists. This will also override the CSV file. Overwrite? (y/N): ") + if response.lower() != 'y': + print("Aborted.") + sys.exit(1) + shutil.rmtree(dest_dir) + # Also remove CSV file if it exists + if os.path.exists(csv_path): + os.remove(csv_path) + + os.makedirs(dest_dir) + + # Scan all files + print(f"Scanning test files in {source_dir}...") + all_files = scan_test_files(source_dir) + print(f"Found {len(all_files)} files total") + + # Detect new tests if --new-since specified + new_uids = None + if args.new_since: + print(f"Detecting new tests since {args.new_since}...") + baseline_uids = detect_new_tests(args.new_since) + + # Get current UIDs + current_uids = _extract_all_uids(all_files) + + # Calculate new UIDs + new_uids = current_uids - baseline_uids + print(f"Current version: {len(current_uids)} tests") + print(f"New tests detected: {len(new_uids)} tests") + + # Stage 1: Component filtering (file-level) with new-since support + if args.component: + print(f"Filtering by components: {args.component}") + component_filtered = filter_by_component(all_files, args.component, new_uids) + else: + component_filtered = all_files + + print(f"After component filter: {len(component_filtered)} files") + + # Stage 2: Content-level filtering + copying + if args.rbp or args.new_since: + filter_desc = [] + if args.rbp: + filter_desc.append(f"RBP: {args.rbp}") + if args.new_since: + filter_desc.append(f"new since {args.new_since}") + print(f"Filtering content by {' + '.join(filter_desc)}") + copied_count = filter_by_rbp_content(component_filtered, args.rbp, dest_dir, args.component, new_uids) + else: + print("No content filters - copying files unchanged") + copied_count = copy_files_unchanged(component_filtered, dest_dir, args.component) + + print(f"\n✓ Setup complete! {copied_count} files copied to {dest_dir}") + + # Generate CSV template after successful setup + print("\nGenerating CSV template...") + try: + import subprocess + script_dir = os.path.dirname(os.path.abspath(__file__)) + parse_script = os.path.join(script_dir, 'parseTestResults.py') + + result = subprocess.run([ + 'python3', parse_script, args.version, '--force' + ], capture_output=True, text=True, check=True) + + print("✓ CSV template generated successfully") + + except subprocess.CalledProcessError as e: + print(f"Warning: CSV generation failed: {e.stderr.strip()}") + print("You can manually generate it later with:") + print(f" python3 parseTestResults.py {args.version}") + except Exception as e: + print(f"Warning: Could not generate CSV template: {e}") + print("You can manually generate it later with:") + print(f" python3 parseTestResults.py {args.version}") + + +if __name__ == "__main__": + main() \ No newline at end of file