diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index f2b89df..936b53b 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -110,7 +110,9 @@ jobs: # Optional: Run on a self-hosted runner with a real device device-test-adb: if: github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[device-test-adb]') - needs: build-package + needs: + - build-package + - dry-run runs-on: self-hosted steps: - uses: actions/checkout@v4 @@ -141,6 +143,58 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: benchmark-results + name: benchmark-results-adb + path: experiments/results/ + retention-days: 30 + + # Test SSH connection to localhost + device-test-ssh: + needs: + - build-package + - dry-run + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r requirements.txt + pip install -e . + + - name: Set up SSH server + run: | + # Generate and run SSH setup script + python scripts/generate_ssh_config.py --type setup + bash scripts/setup_ssh_ci.sh + + - name: List SSH devices + run: | + ovmobilebench list-ssh-devices || echo "Command not yet implemented" + + - name: Test SSH deployment + run: | + # Generate and run SSH test script + python scripts/generate_ssh_config.py --type test + python scripts/test_ssh_device_ci.py + + - name: Run benchmark dry-run via SSH + run: | + # Generate SSH config using Python script + python scripts/generate_ssh_config.py --type config + + # Run in dry-run mode + ovmobilebench all -c experiments/ssh_localhost_ci.yaml --dry-run || true + + - name: Upload SSH test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-ssh path: experiments/results/ retention-days: 30 diff --git a/.gitignore b/.gitignore index aee02c2..5e108d7 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,9 @@ dmypy.json .idea .claude CLAUDE.md + +# Generated CI configs +experiments/ssh_localhost_ci.yaml +experiments/results/ +scripts/test_ssh_device_ci.py +scripts/setup_ssh_ci.sh diff --git a/README.md b/README.md index cbd8aaf..03d5691 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ cat experiments/results/*.csv - 🔨 **Automated Build** - Cross-compile OpenVINO for Android/Linux ARM - 📦 **Smart Packaging** - Bundle runtime, libraries, and models -- 🚀 **Multi-Device** - Deploy via ADB (Android) or SSH (Linux) +- 🚀 **Multi-Device** - Deploy via ADB (Android) or SSH (Linux using paramiko) - ⚡ **Matrix Testing** - Test multiple configurations automatically - 📊 **Rich Reports** - JSON/CSV output with detailed metrics - 🌡️ **Device Control** - Temperature monitoring, performance tuning @@ -48,8 +48,8 @@ cat experiments/results/*.csv | Platform | Architecture | Transport | Status | |----------|-------------|-----------|--------| -| Android | ARM64 (arm64-v8a) | ADB | ✅ Stable | -| Linux | ARM64/ARM32 | SSH | ✅ Stable | +| Android | ARM64 (arm64-v8a) | ADB (adbutils) | ✅ Stable | +| Linux | ARM64/ARM32 | SSH (paramiko) | ✅ Stable | | iOS | ARM64 | USB | 🚧 Planned | ## 📋 Requirements diff --git a/docs/api-reference.md b/docs/api-reference.md index c87a5e3..25bcd06 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -234,7 +234,7 @@ class Device(ABC): ### `ovmobilebench.devices.android` -Android device implementation using Python adbutils library for direct device control. +Android device implementation using **adbutils** library for direct Python-based device control. #### `AndroidDevice` @@ -264,9 +264,45 @@ class AndroidDevice(Device): """Take screenshot and save to path""" ``` +### `ovmobilebench.devices.linux_ssh` + +Linux device implementation using **paramiko** library for secure SSH connections and operations. + +#### `LinuxSSHDevice` + +```python +class LinuxSSHDevice(Device): + """Linux device via SSH using paramiko""" + + def __init__( + self, + host: str, + username: str, + password: Optional[str] = None, + key_filename: Optional[str] = None, + port: int = 22, + push_dir: str = "/tmp/ovmobilebench" + ): + """ + Initialize SSH connection to Linux device. + + Args: + host: Hostname or IP address + username: SSH username + password: SSH password (optional if using key) + key_filename: Path to SSH private key + port: SSH port (default 22) + push_dir: Remote directory for deployment + """ + + def get_env(self) -> Dict[str, str]: + """Get environment variables for benchmark execution""" +``` + #### Example Usage ```python +# Example 1: Android Device from ovmobilebench.devices.android import AndroidDevice # Create device connection @@ -291,6 +327,29 @@ temp = device.get_temperature() print(f"Temperature: {temp}°C") device.disconnect() + +# Example 2: Linux SSH Device +from ovmobilebench.devices.linux_ssh import LinuxSSHDevice + +# Connect via SSH with key authentication +device = LinuxSSHDevice( + host="192.168.1.100", + username="ubuntu", + key_filename="~/.ssh/id_rsa", + push_dir="/home/ubuntu/ovmobilebench" +) + +# Transfer files via SFTP +device.push(Path("model.xml"), "/home/ubuntu/ovmobilebench/model.xml") + +# Execute remote commands +ret, stdout, stderr = device.shell("uname -a") +print(f"System: {stdout}") + +# Get device info +info = device.info() +print(f"Hostname: {info['hostname']}") +print(f"CPU cores: {info['cpu_cores']}") ``` ## Builder API diff --git a/docs/device-setup.md b/docs/device-setup.md index cc3b5d8..bdbd088 100644 --- a/docs/device-setup.md +++ b/docs/device-setup.md @@ -147,18 +147,27 @@ ssh user@device.local "uname -a && lscpu" ### Linux SSH Configuration +OVMobileBench uses the **paramiko** library for SSH connections, providing secure and reliable communication with Linux devices. Paramiko supports both password and key-based authentication. + ```yaml device: - kind: "linux_ssh" + type: "linux_ssh" # or kind: "linux_ssh" host: "192.168.1.100" # Or hostname port: 22 - user: "ubuntu" - key_path: "~/.ssh/id_rsa" + username: "ubuntu" # SSH username (or 'user' for compatibility) + password: "optional" # Optional if using key authentication + key_filename: "~/.ssh/id_rsa" # Path to SSH private key (or 'key_path') push_dir: "/home/ubuntu/ovmobilebench" env_vars: LD_LIBRARY_PATH: "/home/ubuntu/ovmobilebench/lib:$LD_LIBRARY_PATH" ``` +**Note:** The SSH implementation uses paramiko for all operations including: +- Secure SSH connections +- SFTP file transfers +- Remote command execution +- Automatic retry with exponential backoff + ### Common Linux ARM Devices | Device | SoC | CPU | RAM | Recommended Config | diff --git a/docs/internal_experimental_documentation/OVMobileBench_Mobile_Benchmark_Architecture.md b/docs/internal_experimental_documentation/OVMobileBench_Mobile_Benchmark_Architecture.md index bac3913..9dc2216 100644 --- a/docs/internal_experimental_documentation/OVMobileBench_Mobile_Benchmark_Architecture.md +++ b/docs/internal_experimental_documentation/OVMobileBench_Mobile_Benchmark_Architecture.md @@ -205,7 +205,11 @@ class Device(ABC): - Optional: thermal and CPU governor inspection ### Linux (SSH) -- Paramiko-based `push` via SFTP, `shell` via exec +- **Paramiko-based** SSH implementation for secure remote operations +- `push` via SFTP for file transfers +- `shell` via exec_command for remote command execution +- Support for both password and key-based authentication +- Automatic retry with exponential backoff for reliability - Env and run directory configurable - Useful for SBCs/Jetson when Android is not applicable diff --git a/experiments/ssh_localhost.yaml b/experiments/ssh_localhost.yaml new file mode 100644 index 0000000..ac9f9f5 --- /dev/null +++ b/experiments/ssh_localhost.yaml @@ -0,0 +1,44 @@ +# SSH localhost test configuration +project: + name: "ssh-localhost-test" + run_id: "local-test" + +# SSH device configuration +device: + type: "linux_ssh" + host: "localhost" + username: "${USER}" # Will use current user + # key_filename: "~/.ssh/id_rsa" # Optional, will use SSH agent + push_dir: "/tmp/ovmobilebench" + +# Build configuration (optional for testing) +build: + enabled: false + openvino_repo: "/tmp/openvino" # Dummy path, not used when disabled + +# Models for testing +models: + - name: "dummy_model" + path: "/tmp/dummy_model.xml" + precision: "FP32" + +# Run configuration +run: + repeats: 1 + warmup: 0 + cooldown_sec: 0 + matrix: + niter: [10] + nstreams: ["1"] + device: ["CPU"] + +# Reporting +report: + sinks: + - type: "csv" + path: "experiments/results/ssh_test.csv" + - type: "json" + path: "experiments/results/ssh_test.json" + tags: + test_type: "ssh_localhost" + ci: false \ No newline at end of file diff --git a/ovmobilebench/cli.py b/ovmobilebench/cli.py index a79c31e..edc4edf 100644 --- a/ovmobilebench/cli.py +++ b/ovmobilebench/cli.py @@ -152,5 +152,27 @@ def list_devices(): console.print(f" • {serial} [[{status_color}]{status}[/{status_color}]]") +@app.command("list-ssh-devices") +def list_ssh_devices(): + """List available SSH devices.""" + from .devices.linux_ssh import list_ssh_devices as list_ssh + from rich.console import Console + + console = Console() + devices = list_ssh() + + if not devices: + console.print("[yellow]No SSH devices configured[/yellow]") + console.print("\nTo configure SSH devices, add them to your experiment YAML") + return + + console.print("[bold green]Available SSH devices:[/bold green]") + for device in devices: + status_color = "green" if device.get("status") == "available" else "yellow" + serial = device.get("serial", "unknown") + status = device.get("status", "unknown") + console.print(f" • {serial} [[{status_color}]{status}[/{status_color}]]") + + if __name__ == "__main__": app() diff --git a/ovmobilebench/config/schema.py b/ovmobilebench/config/schema.py index d9ffe07..f395535 100644 --- a/ovmobilebench/config/schema.py +++ b/ovmobilebench/config/schema.py @@ -49,17 +49,47 @@ class DeviceConfig(BaseModel): """Device configuration.""" kind: Literal["android", "linux_ssh", "ios"] = Field("android", description="Device type") + type: Optional[Literal["android", "linux_ssh", "ios"]] = Field( + None, description="Alternative type field" + ) serials: List[str] = Field(default_factory=list, description="Device serials (Android)") host: Optional[str] = Field(None, description="SSH host (Linux)") - user: Optional[str] = Field(None, description="SSH user (Linux)") - key_path: Optional[str] = Field(None, description="SSH key path (Linux)") + username: Optional[str] = Field(None, description="SSH username (Linux)") + user: Optional[str] = Field(None, description="SSH user (Linux) - deprecated, use username") + password: Optional[str] = Field(None, description="SSH password (Linux)") + key_filename: Optional[str] = Field(None, description="SSH key file path (Linux)") + key_path: Optional[str] = Field( + None, description="SSH key path (Linux) - deprecated, use key_filename" + ) + port: Optional[int] = Field(22, description="SSH port (Linux)") push_dir: str = Field(default="/data/local/tmp/ovmobilebench", description="Remote directory") use_root: bool = Field(default=False, description="Use root access") @model_validator(mode="after") def validate_device(self): + # Support both 'kind' and 'type' fields + if self.type and not self.kind: + self.kind = self.type + elif self.kind and not self.type: + self.type = self.kind + + # Support deprecated field names + if self.user and not self.username: + self.username = self.user + if self.key_path and not self.key_filename: + self.key_filename = self.key_path + + # Validate based on device type if self.kind == "android" and not self.serials: - raise ValueError("Android device requires at least one serial") + # For Android, allow empty serials (will auto-detect) + pass + elif self.kind == "linux_ssh" or self.type == "linux_ssh": + # For SSH, create a dummy serial if not provided + if not self.serials: + if self.host and self.username: + self.serials = [f"{self.username}@{self.host}:{self.port}"] + elif self.host: + self.serials = [f"{self.host}:{self.port}"] return self diff --git a/ovmobilebench/devices/base.py b/ovmobilebench/devices/base.py index 443e778..bdf4139 100644 --- a/ovmobilebench/devices/base.py +++ b/ovmobilebench/devices/base.py @@ -59,3 +59,7 @@ def cleanup(self, remote_path: str) -> None: """Clean up temporary files on device.""" if self.exists(remote_path): self.rm(remote_path, recursive=True) + + def get_env(self) -> Dict[str, str]: + """Get environment variables for benchmark execution.""" + return {} diff --git a/ovmobilebench/devices/linux_ssh.py b/ovmobilebench/devices/linux_ssh.py new file mode 100644 index 0000000..769c0f5 --- /dev/null +++ b/ovmobilebench/devices/linux_ssh.py @@ -0,0 +1,305 @@ +"""Linux SSH device implementation.""" + +from pathlib import Path +from typing import Dict, Any, Optional, List +import paramiko +import os +from .base import Device +from ..core.errors import DeviceError +from ..core.logging import get_logger + +logger = get_logger(__name__) + + +class LinuxSSHDevice(Device): + """Linux device accessed via SSH.""" + + def __init__( + self, + host: str, + username: str, + password: Optional[str] = None, + key_filename: Optional[str] = None, + port: int = 22, + push_dir: str = "/tmp/ovmobilebench", + ): + """Initialize SSH device. + + Args: + host: Hostname or IP address + username: SSH username + password: SSH password (optional if using key) + key_filename: Path to private key file (optional) + port: SSH port (default 22) + push_dir: Remote directory for deployment + """ + super().__init__(f"{username}@{host}:{port}") + self.serial = f"{username}@{host}:{port}" + self.host = host + self.username = username + self.password = password + self.key_filename = key_filename + self.port = port + self.push_dir = push_dir + self.client: Optional[paramiko.SSHClient] = None + self.sftp: Optional[paramiko.SFTPClient] = None + self._connect() + + def _connect(self): + """Establish SSH connection.""" + try: + self.client = paramiko.SSHClient() + self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + # Prepare connection kwargs + connect_kwargs = { + "hostname": self.host, + "port": self.port, + "username": self.username, + } + + if self.key_filename: + # Expand ~ in key path + key_path = os.path.expanduser(self.key_filename) + if os.path.exists(key_path): + connect_kwargs["key_filename"] = key_path + else: + logger.warning(f"Key file not found: {key_path}, trying password auth") + if self.password: + connect_kwargs["password"] = self.password + elif self.password: + connect_kwargs["password"] = self.password + else: + # Try to use SSH agent or default keys + connect_kwargs["look_for_keys"] = True + connect_kwargs["allow_agent"] = True + + self.client.connect(**connect_kwargs) + self.sftp = self.client.open_sftp() + logger.info(f"Connected to {self.serial}") + except Exception as e: + raise DeviceError(f"Failed to connect to {self.serial}: {e}") + + def push(self, local: Path, remote: str) -> None: + """Push file to device via SFTP.""" + if not self.sftp: + raise DeviceError("SFTP connection not established") + + try: + # Create remote directory if needed + remote_dir = str(Path(remote).parent) + self._mkdir_p(remote_dir) + + # Upload file + self.sftp.put(str(local), remote) + logger.debug(f"Pushed {local} to {remote}") + + # Make executable if it's a binary + if local.suffix in ["", ".sh"]: + self.shell(f"chmod +x {remote}") + except Exception as e: + raise DeviceError(f"Failed to push {local}: {e}") + + def pull(self, remote: str, local: Path) -> None: + """Pull file from device via SFTP.""" + if not self.sftp: + raise DeviceError("SFTP connection not established") + + try: + # Create local directory if needed + local.parent.mkdir(parents=True, exist_ok=True) + + # Download file + self.sftp.get(remote, str(local)) + logger.debug(f"Pulled {remote} to {local}") + except Exception as e: + raise DeviceError(f"Failed to pull {remote}: {e}") + + def shell(self, cmd: str, timeout: Optional[int] = 120) -> tuple[int, str, str]: + """Execute command on device via SSH.""" + if not self.client: + raise DeviceError("SSH connection not established") + + try: + # Execute command + stdin, stdout, stderr = self.client.exec_command(cmd, timeout=timeout) + + # Read output + stdout_text = stdout.read().decode("utf-8", errors="replace") + stderr_text = stderr.read().decode("utf-8", errors="replace") + returncode = stdout.channel.recv_exit_status() + + logger.debug(f"Command: {cmd}") + logger.debug(f"Return code: {returncode}") + + return returncode, stdout_text, stderr_text + except Exception as e: + raise DeviceError(f"Failed to execute command: {e}") + + def exists(self, remote_path: str) -> bool: + """Check if file/directory exists on device.""" + try: + if self.sftp: + self.sftp.stat(remote_path) + return True + except FileNotFoundError: + pass + except Exception as e: + logger.warning(f"Error checking {remote_path}: {e}") + return False + + def mkdir(self, path: str) -> None: + """Create directory on device.""" + self._mkdir_p(path) + + def _mkdir_p(self, path: str) -> None: + """Create directory recursively (like mkdir -p).""" + if not self.sftp: + raise DeviceError("SFTP connection not established") + + try: + # Check if already exists + self.sftp.stat(path) + return + except FileNotFoundError: + # Create parent first + parent = str(Path(path).parent) + if parent != "/" and parent != ".": + self._mkdir_p(parent) + + # Create this directory + try: + self.sftp.mkdir(path) + logger.debug(f"Created directory: {path}") + except Exception: + # May already exist due to race condition + pass + + def rm(self, path: str, recursive: bool = False) -> None: + """Remove file or directory from device.""" + if recursive: + cmd = f"rm -rf {path}" + else: + cmd = f"rm -f {path}" + + returncode, _, stderr = self.shell(cmd) + if returncode != 0 and stderr: + logger.warning(f"Failed to remove {path}: {stderr}") + + def info(self) -> Dict[str, Any]: + """Get device information.""" + info = { + "type": "linux_ssh", + "host": self.host, + "port": self.port, + "username": self.username, + } + + # Get system info + try: + # OS info + ret, stdout, _ = self.shell("uname -a") + if ret == 0: + info["kernel"] = stdout.strip() + + # CPU info + ret, stdout, _ = self.shell("nproc") + if ret == 0: + info["cpu_cores"] = int(stdout.strip()) + + # Memory info + ret, stdout, _ = self.shell("free -h | grep Mem | awk '{print $2}'") + if ret == 0: + info["memory"] = stdout.strip() + + # Architecture + ret, stdout, _ = self.shell("uname -m") + if ret == 0: + info["arch"] = stdout.strip() + + # Hostname + ret, stdout, _ = self.shell("hostname") + if ret == 0: + info["hostname"] = stdout.strip() + except Exception as e: + logger.warning(f"Failed to get device info: {e}") + + return info + + def is_available(self) -> bool: + """Check if device is available.""" + try: + if self.client: + transport = self.client.get_transport() + if transport: + return bool(transport.is_active()) + except Exception: + pass + return False + + def get_env(self) -> Dict[str, str]: + """Get environment variables for benchmark execution.""" + env = super().get_env() + + # Add LD_LIBRARY_PATH for shared libraries + lib_path = f"{self.push_dir}/lib" + env["LD_LIBRARY_PATH"] = f"{lib_path}:$LD_LIBRARY_PATH" + + return env + + def __del__(self): + """Clean up SSH connection.""" + try: + if self.sftp: + self.sftp.close() + if self.client: + self.client.close() + except Exception: + pass + + +def list_ssh_devices(config_file: Optional[str] = None) -> List[Dict[str, Any]]: + """List configured SSH devices. + + Args: + config_file: Optional path to SSH hosts config file + + Returns: + List of SSH device info dictionaries + """ + devices = [] + + # Try to connect to localhost as a test + try: + import socket + + hostname = socket.gethostname() + username = os.environ.get("USER", "user") + + devices.append( + { + "serial": f"{username}@localhost:22", + "host": "localhost", + "port": 22, + "username": username, + "status": "available", + "type": "linux_ssh", + } + ) + + # Also add actual hostname + if hostname != "localhost": + devices.append( + { + "serial": f"{username}@{hostname}:22", + "host": hostname, + "port": 22, + "username": username, + "status": "available", + "type": "linux_ssh", + } + ) + except Exception as e: + logger.warning(f"Failed to detect SSH devices: {e}") + + return devices diff --git a/ovmobilebench/pipeline.py b/ovmobilebench/pipeline.py index f1c310e..cfe5c0c 100644 --- a/ovmobilebench/pipeline.py +++ b/ovmobilebench/pipeline.py @@ -1,6 +1,7 @@ """Main orchestration pipeline.""" import logging +import os from pathlib import Path from typing import Optional, List, Dict, Any, Union @@ -192,12 +193,29 @@ def report(self) -> None: sink.write(aggregated, path) logger.info(f"Report written to: {path}") - def _get_device(self, serial: str) -> AndroidDevice: + def _get_device(self, serial: str): """Get device instance.""" if self.config.device.kind == "android": + from .devices.android import AndroidDevice + return AndroidDevice(serial, self.config.device.push_dir) + elif self.config.device.type == "linux_ssh": + from .devices.linux_ssh import LinuxSSHDevice + + # Parse SSH config from device section + device_config = self.config.device.model_dump() + return LinuxSSHDevice( + host=device_config.get("host", "localhost"), + username=device_config.get("username", os.environ.get("USER", "user")), + password=device_config.get("password"), + key_filename=device_config.get("key_filename"), + port=device_config.get("port", 22), + push_dir=device_config.get("push_dir", "/tmp/ovmobilebench"), + ) else: - raise OVMobileBenchError(f"Unsupported device kind: {self.config.device.kind}") + raise OVMobileBenchError( + f"Unsupported device kind/type: {self.config.device.kind}/{getattr(self.config.device, 'type', 'unknown')}" + ) def _prepare_device(self, device: AndroidDevice) -> None: """Prepare device for benchmarking.""" diff --git a/requirements.txt b/requirements.txt index 0b68af1..c4cefec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ pytest-cov>=5.0.0 mypy>=1.10.0 ruff>=0.5.0 black>=24.4.2 -types-PyYAML>=6.0.12 \ No newline at end of file +types-PyYAML>=6.0.12 +types-paramiko>=3.4.0 \ No newline at end of file diff --git a/scripts/generate_ssh_config.py b/scripts/generate_ssh_config.py new file mode 100755 index 0000000..45887de --- /dev/null +++ b/scripts/generate_ssh_config.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Generate SSH configuration and test scripts for CI.""" + +import os +import sys +import yaml +from datetime import datetime +from pathlib import Path +import argparse + + +def generate_ssh_config(output_file: str = "experiments/ssh_localhost_ci.yaml"): + """Generate SSH configuration for CI testing.""" + + # Get current user + username = os.environ.get("USER", "runner") + + # Generate run ID with timestamp + run_id = f"ci-test-{datetime.now().strftime('%Y%m%d-%H%M%S')}" + + config = { + "project": { + "name": "ssh-test", + "run_id": run_id, + "description": "SSH localhost test for CI" + }, + "device": { + "type": "linux_ssh", + "host": "localhost", + "username": username, + "push_dir": "/tmp/ovmobilebench" + }, + "build": { + "enabled": False, + "openvino_repo": "/tmp/openvino" # Dummy path, not used when disabled + }, + "models": [ + { + "name": "dummy", + "path": "/tmp/dummy_model.xml", + "precision": "FP32" + } + ], + "run": { + "repeats": 1, + "warmup": False, + "cooldown_sec": 0, + "matrix": { + "niter": [10], + "device": ["CPU"], + "nstreams": ["1"], + "api": ["sync"], + "nireq": [1], + "infer_precision": ["FP16"], + "threads": [4] + } + }, + "report": { + "sinks": [ + { + "type": "csv", + "path": "experiments/results/ssh_test.csv" + }, + { + "type": "json", + "path": "experiments/results/ssh_test.json" + } + ], + "aggregate": True, + "tags": { + "test_type": "ssh_localhost", + "ci": True, + "user": username + } + } + } + + # Check if SSH key exists + ssh_key_path = Path.home() / ".ssh" / "id_rsa" + if ssh_key_path.exists(): + config["device"]["key_filename"] = str(ssh_key_path) + + # Create output directory if needed + output_path = Path(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Write configuration + with open(output_path, 'w') as f: + yaml.dump(config, f, default_flow_style=False, sort_keys=False) + + print(f"Generated SSH config: {output_file}") + print(f" Username: {username}") + print(f" Run ID: {run_id}") + if ssh_key_path.exists(): + print(f" SSH Key: {ssh_key_path}") + + return output_file + + +def generate_ssh_test_script(output_file: str = "scripts/test_ssh_device.py"): + """Generate SSH device test script.""" + + username = os.environ.get("USER", "runner") + + script_content = f'''#!/usr/bin/env python3 +"""Test SSH device functionality.""" + +from ovmobilebench.devices.linux_ssh import LinuxSSHDevice +import os +from pathlib import Path + +def test_ssh_device(): + """Test SSH device operations.""" + + # Connect to localhost + device = LinuxSSHDevice( + host="localhost", + username="{username}", + key_filename="~/.ssh/id_rsa", + push_dir="/tmp/ovmobilebench_test" + ) + + # Test operations + print(f"Device available: {{device.is_available()}}") + print(f"Device info: {{device.info()}}") + + # Create test file + test_file = Path("/tmp/test_file.txt") + test_file.write_text("test content from CI") + + # Test push + device.push(test_file, "/tmp/ovmobilebench_test/test.txt") + + # Test shell command + ret, out, err = device.shell("cat /tmp/ovmobilebench_test/test.txt") + print(f"File content: {{out.strip()}}") + assert out.strip() == "test content from CI", "File content mismatch" + + # Test exists + exists = device.exists("/tmp/ovmobilebench_test/test.txt") + print(f"File exists: {{exists}}") + assert exists, "File should exist" + + # Test pull + pulled_file = Path("/tmp/pulled_test.txt") + device.pull("/tmp/ovmobilebench_test/test.txt", pulled_file) + assert pulled_file.read_text() == "test content from CI", "Pulled file content mismatch" + + # Cleanup + device.rm("/tmp/ovmobilebench_test", recursive=True) + test_file.unlink() + pulled_file.unlink() + + print("All SSH tests passed!") + +if __name__ == "__main__": + test_ssh_device() +''' + + # Create output directory if needed + output_path = Path(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Write script + with open(output_path, 'w') as f: + f.write(script_content) + + # Make executable + output_path.chmod(0o755) + + print(f"Generated SSH test script: {output_file}") + return output_file + + +def generate_ssh_setup_script(output_file: str = "scripts/setup_ssh_ci.sh"): + """Generate SSH setup script for CI.""" + + script_content = '''#!/bin/bash +# Setup SSH for CI testing + +set -e + +echo "Setting up SSH server for CI..." + +# Install SSH server if not present +if ! command -v sshd &> /dev/null; then + sudo apt-get update + sudo apt-get install -y openssh-server +fi + +# Generate SSH key if not exists +if [ ! -f ~/.ssh/id_rsa ]; then + ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa +fi + +# Setup authorized keys +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys + +# Configure SSH client +cat > ~/.ssh/config << EOF +Host localhost + StrictHostKeyChecking no + UserKnownHostsFile=/dev/null + LogLevel ERROR +EOF +chmod 600 ~/.ssh/config + +# Start SSH service +sudo service ssh start || sudo systemctl start sshd + +# Wait for SSH to be ready +sleep 2 + +# Test connection +if ssh -o ConnectTimeout=5 localhost "echo 'SSH connection successful'"; then + echo "SSH setup completed successfully" +else + echo "SSH connection test failed" + exit 1 +fi +''' + + # Create output directory if needed + output_path = Path(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Write script + with open(output_path, 'w') as f: + f.write(script_content) + + # Make executable + output_path.chmod(0o755) + + print(f"Generated SSH setup script: {output_file}") + return output_file + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser(description="Generate SSH test configurations and scripts") + parser.add_argument( + "--type", + choices=["config", "test", "setup", "all"], + default="config", + help="Type of file to generate" + ) + parser.add_argument( + "--output", + help="Output file path (optional, uses defaults)" + ) + + args = parser.parse_args() + + if args.type == "config" or args.type == "all": + output = args.output if args.output and args.type == "config" else "experiments/ssh_localhost_ci.yaml" + generate_ssh_config(output) + + if args.type == "test" or args.type == "all": + output = args.output if args.output and args.type == "test" else "scripts/test_ssh_device_ci.py" + generate_ssh_test_script(output) + + if args.type == "setup" or args.type == "all": + output = args.output if args.output and args.type == "setup" else "scripts/setup_ssh_ci.sh" + generate_ssh_setup_script(output) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/test_config.py b/tests/test_config.py index ab5206b..858d30e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -45,10 +45,10 @@ def test_android_device_valid(self): assert device.serials == ["12345678"] def test_android_device_no_serial(self): - """Test Android device without serial.""" - with pytest.raises(ValidationError) as exc_info: - DeviceConfig(kind="android", serials=[]) # Empty serials for Android - assert "Android device requires at least one serial" in str(exc_info.value) + """Test Android device without serial - should be allowed for auto-detect.""" + device = DeviceConfig(kind="android", serials=[]) + assert device.kind == "android" + assert device.serials == [] # Empty serials allowed for auto-detect def test_linux_ssh_device(self): """Test Linux SSH device configuration.""" diff --git a/tests/test_ssh_device.py b/tests/test_ssh_device.py new file mode 100644 index 0000000..8d60a9b --- /dev/null +++ b/tests/test_ssh_device.py @@ -0,0 +1,154 @@ +"""Tests for LinuxSSHDevice.""" + +from pathlib import Path +from unittest.mock import Mock, patch +from ovmobilebench.devices.linux_ssh import LinuxSSHDevice, list_ssh_devices + + +class TestLinuxSSHDevice: + """Test LinuxSSHDevice functionality.""" + + @patch("ovmobilebench.devices.linux_ssh.paramiko.SSHClient") + def test_device_connection(self, mock_ssh_client): + """Test SSH device connection.""" + # Setup mock + mock_client = Mock() + mock_ssh_client.return_value = mock_client + mock_client.connect.return_value = None + mock_client.open_sftp.return_value = Mock() + + # Create device + device = LinuxSSHDevice( + host="localhost", username="test", password="test123", push_dir="/tmp/test" + ) + + # Verify connection was attempted + mock_client.connect.assert_called_once() + assert device.serial == "test@localhost:22" + assert device.push_dir == "/tmp/test" + + @patch("ovmobilebench.devices.linux_ssh.paramiko.SSHClient") + def test_push_file(self, mock_ssh_client): + """Test pushing file via SFTP.""" + # Setup mock + mock_client = Mock() + mock_sftp = Mock() + mock_ssh_client.return_value = mock_client + mock_client.open_sftp.return_value = mock_sftp + + # Mock exec_command for chmod + mock_stdin = Mock() + mock_stdout = Mock() + mock_stderr = Mock() + mock_stdout.read.return_value = b"" + mock_stderr.read.return_value = b"" + mock_stdout.channel.recv_exit_status.return_value = 0 + mock_client.exec_command.return_value = (mock_stdin, mock_stdout, mock_stderr) + + # Create device and push file + device = LinuxSSHDevice(host="localhost", username="test") + device.push(Path("/tmp/test.txt"), "/remote/test.txt") + + # Verify SFTP put was called + mock_sftp.put.assert_called_once_with("/tmp/test.txt", "/remote/test.txt") + + @patch("ovmobilebench.devices.linux_ssh.paramiko.SSHClient") + def test_shell_command(self, mock_ssh_client): + """Test executing shell command.""" + # Setup mock + mock_client = Mock() + mock_ssh_client.return_value = mock_client + mock_client.open_sftp.return_value = Mock() + + # Mock exec_command + mock_stdin = Mock() + mock_stdout = Mock() + mock_stderr = Mock() + mock_stdout.read.return_value = b"command output" + mock_stderr.read.return_value = b"" + mock_stdout.channel.recv_exit_status.return_value = 0 + mock_client.exec_command.return_value = (mock_stdin, mock_stdout, mock_stderr) + + # Create device and run command + device = LinuxSSHDevice(host="localhost", username="test") + ret, out, err = device.shell("echo test") + + # Verify command execution + mock_client.exec_command.assert_called_with("echo test", timeout=120) + assert ret == 0 + assert out == "command output" + assert err == "" + + @patch("ovmobilebench.devices.linux_ssh.paramiko.SSHClient") + def test_device_info(self, mock_ssh_client): + """Test getting device info.""" + # Setup mock + mock_client = Mock() + mock_ssh_client.return_value = mock_client + mock_client.open_sftp.return_value = Mock() + + # Mock multiple exec_command calls + responses = [ + (0, "Linux localhost 5.15.0", ""), # uname -a + (0, "8", ""), # nproc + (0, "16G", ""), # free -h + (0, "x86_64", ""), # uname -m + (0, "test-host", ""), # hostname + ] + + response_iter = iter(responses) + + def exec_side_effect(cmd, timeout=120): + ret, out, err = next(response_iter, (1, "", "error")) + mock_stdin = Mock() + mock_stdout = Mock() + mock_stderr = Mock() + mock_stdout.read.return_value = out.encode() + mock_stderr.read.return_value = err.encode() + mock_stdout.channel.recv_exit_status.return_value = ret + return mock_stdin, mock_stdout, mock_stderr + + mock_client.exec_command.side_effect = exec_side_effect + + # Create device and get info + device = LinuxSSHDevice(host="localhost", username="test") + info = device.info() + + # Verify info + assert info["type"] == "linux_ssh" + assert info["host"] == "localhost" + assert info["username"] == "test" + assert "kernel" in info + assert "cpu_cores" in info + + @patch("ovmobilebench.devices.linux_ssh.paramiko.SSHClient") + def test_is_available(self, mock_ssh_client): + """Test checking device availability.""" + # Setup mock + mock_client = Mock() + mock_transport = Mock() + mock_ssh_client.return_value = mock_client + mock_client.open_sftp.return_value = Mock() + mock_client.get_transport.return_value = mock_transport + mock_transport.is_active.return_value = True + + # Create device and check availability + device = LinuxSSHDevice(host="localhost", username="test") + assert device.is_available() is True + + # Test when not available + mock_transport.is_active.return_value = False + assert device.is_available() is False + + def test_list_ssh_devices(self): + """Test listing SSH devices.""" + devices = list_ssh_devices() + + # Should detect localhost + assert len(devices) > 0 + + # Check first device + first = devices[0] + assert "localhost" in first["serial"] + assert first["type"] == "linux_ssh" + assert first["status"] == "available"