From fc919b808c6657c4db091c13bee0edf731bdee36 Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Sat, 16 Aug 2025 01:28:28 +0200 Subject: [PATCH 1/5] add SSH device support using paramiko, update CLI, schemas, tests, and documentation --- .github/workflows/bench.yml | 128 +++++++- README.md | 6 +- docs/api-reference.md | 61 +++- docs/device-setup.md | 15 +- ...bileBench_Mobile_Benchmark_Architecture.md | 6 +- experiments/ssh_localhost.yaml | 44 +++ ovmobilebench/cli.py | 22 ++ ovmobilebench/config/schema.py | 32 +- ovmobilebench/devices/base.py | 4 + ovmobilebench/devices/linux_ssh.py | 298 ++++++++++++++++++ ovmobilebench/pipeline.py | 18 +- tests/test_ssh_device.py | 159 ++++++++++ 12 files changed, 779 insertions(+), 14 deletions(-) create mode 100644 experiments/ssh_localhost.yaml create mode 100644 ovmobilebench/devices/linux_ssh.py create mode 100644 tests/test_ssh_device.py diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index f2b89df..b4a2fab 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -141,6 +141,132 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: benchmark-results + name: benchmark-results-adb + path: experiments/results/ + retention-days: 30 + + # Test SSH connection to localhost + device-test-ssh: + if: github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[device-test-ssh]') + needs: build-package + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r requirements.txt + pip install -e . + + - name: Set up SSH server + run: | + # Install SSH server if not present + sudo apt-get update + sudo apt-get install -y openssh-server + + # Configure SSH for passwordless localhost access + ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa || true + cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys + chmod 600 ~/.ssh/authorized_keys + + # Configure SSH client + cat > ~/.ssh/config << EOF + Host localhost + StrictHostKeyChecking no + UserKnownHostsFile=/dev/null + EOF + chmod 600 ~/.ssh/config + + # Start SSH service + sudo service ssh start + + # Test connection + ssh -o ConnectTimeout=5 localhost "echo 'SSH connection successful'" + + - name: List SSH devices + run: | + ovmobilebench list-ssh-devices || echo "Command not yet implemented" + + - name: Test SSH deployment + run: | + # Create test file + echo "test content" > /tmp/test_file.txt + + # Test using Python + python << 'EOF' + from ovmobilebench.devices.linux_ssh import LinuxSSHDevice + import os + from pathlib import Path + + # Connect to localhost + device = LinuxSSHDevice( + host="localhost", + username=os.environ["USER"], + key_filename="~/.ssh/id_rsa", + push_dir="/tmp/ovmobilebench_test" + ) + + # Test operations + print(f"Device available: {device.is_available()}") + print(f"Device info: {device.info()}") + + # Test push + device.push(Path("/tmp/test_file.txt"), "/tmp/ovmobilebench_test/test.txt") + + # Test shell command + ret, out, err = device.shell("cat /tmp/ovmobilebench_test/test.txt") + print(f"File content: {out.strip()}") + + # Test exists + print(f"File exists: {device.exists('/tmp/ovmobilebench_test/test.txt')}") + + # Cleanup + device.rm("/tmp/ovmobilebench_test", recursive=True) + EOF + + - name: Run benchmark dry-run via SSH + run: | + # Create minimal config for SSH + cat > experiments/ssh_localhost.yaml << 'EOF' + project: + name: "ssh-test" + run_id: "ci-test" + + device: + type: "linux_ssh" + host: "localhost" + username: "${USER}" + key_filename: "~/.ssh/id_rsa" + push_dir: "/tmp/ovmobilebench" + + models: + - name: "dummy" + path: "/tmp/dummy_model.xml" + + run: + repeats: 1 + matrix: + niter: [10] + + report: + sinks: + - type: "csv" + path: "experiments/results/ssh_test.csv" + EOF + + # Run in dry-run mode + ovmobilebench all -c experiments/ssh_localhost.yaml --dry-run || true + + - name: Upload SSH test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-ssh path: experiments/results/ retention-days: 30 diff --git a/README.md b/README.md index cbd8aaf..03d5691 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ cat experiments/results/*.csv - 🔨 **Automated Build** - Cross-compile OpenVINO for Android/Linux ARM - 📦 **Smart Packaging** - Bundle runtime, libraries, and models -- 🚀 **Multi-Device** - Deploy via ADB (Android) or SSH (Linux) +- 🚀 **Multi-Device** - Deploy via ADB (Android) or SSH (Linux using paramiko) - ⚡ **Matrix Testing** - Test multiple configurations automatically - 📊 **Rich Reports** - JSON/CSV output with detailed metrics - 🌡️ **Device Control** - Temperature monitoring, performance tuning @@ -48,8 +48,8 @@ cat experiments/results/*.csv | Platform | Architecture | Transport | Status | |----------|-------------|-----------|--------| -| Android | ARM64 (arm64-v8a) | ADB | ✅ Stable | -| Linux | ARM64/ARM32 | SSH | ✅ Stable | +| Android | ARM64 (arm64-v8a) | ADB (adbutils) | ✅ Stable | +| Linux | ARM64/ARM32 | SSH (paramiko) | ✅ Stable | | iOS | ARM64 | USB | 🚧 Planned | ## 📋 Requirements diff --git a/docs/api-reference.md b/docs/api-reference.md index c87a5e3..25bcd06 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -234,7 +234,7 @@ class Device(ABC): ### `ovmobilebench.devices.android` -Android device implementation using Python adbutils library for direct device control. +Android device implementation using **adbutils** library for direct Python-based device control. #### `AndroidDevice` @@ -264,9 +264,45 @@ class AndroidDevice(Device): """Take screenshot and save to path""" ``` +### `ovmobilebench.devices.linux_ssh` + +Linux device implementation using **paramiko** library for secure SSH connections and operations. + +#### `LinuxSSHDevice` + +```python +class LinuxSSHDevice(Device): + """Linux device via SSH using paramiko""" + + def __init__( + self, + host: str, + username: str, + password: Optional[str] = None, + key_filename: Optional[str] = None, + port: int = 22, + push_dir: str = "/tmp/ovmobilebench" + ): + """ + Initialize SSH connection to Linux device. + + Args: + host: Hostname or IP address + username: SSH username + password: SSH password (optional if using key) + key_filename: Path to SSH private key + port: SSH port (default 22) + push_dir: Remote directory for deployment + """ + + def get_env(self) -> Dict[str, str]: + """Get environment variables for benchmark execution""" +``` + #### Example Usage ```python +# Example 1: Android Device from ovmobilebench.devices.android import AndroidDevice # Create device connection @@ -291,6 +327,29 @@ temp = device.get_temperature() print(f"Temperature: {temp}°C") device.disconnect() + +# Example 2: Linux SSH Device +from ovmobilebench.devices.linux_ssh import LinuxSSHDevice + +# Connect via SSH with key authentication +device = LinuxSSHDevice( + host="192.168.1.100", + username="ubuntu", + key_filename="~/.ssh/id_rsa", + push_dir="/home/ubuntu/ovmobilebench" +) + +# Transfer files via SFTP +device.push(Path("model.xml"), "/home/ubuntu/ovmobilebench/model.xml") + +# Execute remote commands +ret, stdout, stderr = device.shell("uname -a") +print(f"System: {stdout}") + +# Get device info +info = device.info() +print(f"Hostname: {info['hostname']}") +print(f"CPU cores: {info['cpu_cores']}") ``` ## Builder API diff --git a/docs/device-setup.md b/docs/device-setup.md index cc3b5d8..bdbd088 100644 --- a/docs/device-setup.md +++ b/docs/device-setup.md @@ -147,18 +147,27 @@ ssh user@device.local "uname -a && lscpu" ### Linux SSH Configuration +OVMobileBench uses the **paramiko** library for SSH connections, providing secure and reliable communication with Linux devices. Paramiko supports both password and key-based authentication. + ```yaml device: - kind: "linux_ssh" + type: "linux_ssh" # or kind: "linux_ssh" host: "192.168.1.100" # Or hostname port: 22 - user: "ubuntu" - key_path: "~/.ssh/id_rsa" + username: "ubuntu" # SSH username (or 'user' for compatibility) + password: "optional" # Optional if using key authentication + key_filename: "~/.ssh/id_rsa" # Path to SSH private key (or 'key_path') push_dir: "/home/ubuntu/ovmobilebench" env_vars: LD_LIBRARY_PATH: "/home/ubuntu/ovmobilebench/lib:$LD_LIBRARY_PATH" ``` +**Note:** The SSH implementation uses paramiko for all operations including: +- Secure SSH connections +- SFTP file transfers +- Remote command execution +- Automatic retry with exponential backoff + ### Common Linux ARM Devices | Device | SoC | CPU | RAM | Recommended Config | diff --git a/docs/internal_experimental_documentation/OVMobileBench_Mobile_Benchmark_Architecture.md b/docs/internal_experimental_documentation/OVMobileBench_Mobile_Benchmark_Architecture.md index bac3913..9dc2216 100644 --- a/docs/internal_experimental_documentation/OVMobileBench_Mobile_Benchmark_Architecture.md +++ b/docs/internal_experimental_documentation/OVMobileBench_Mobile_Benchmark_Architecture.md @@ -205,7 +205,11 @@ class Device(ABC): - Optional: thermal and CPU governor inspection ### Linux (SSH) -- Paramiko-based `push` via SFTP, `shell` via exec +- **Paramiko-based** SSH implementation for secure remote operations +- `push` via SFTP for file transfers +- `shell` via exec_command for remote command execution +- Support for both password and key-based authentication +- Automatic retry with exponential backoff for reliability - Env and run directory configurable - Useful for SBCs/Jetson when Android is not applicable diff --git a/experiments/ssh_localhost.yaml b/experiments/ssh_localhost.yaml new file mode 100644 index 0000000..ac9f9f5 --- /dev/null +++ b/experiments/ssh_localhost.yaml @@ -0,0 +1,44 @@ +# SSH localhost test configuration +project: + name: "ssh-localhost-test" + run_id: "local-test" + +# SSH device configuration +device: + type: "linux_ssh" + host: "localhost" + username: "${USER}" # Will use current user + # key_filename: "~/.ssh/id_rsa" # Optional, will use SSH agent + push_dir: "/tmp/ovmobilebench" + +# Build configuration (optional for testing) +build: + enabled: false + openvino_repo: "/tmp/openvino" # Dummy path, not used when disabled + +# Models for testing +models: + - name: "dummy_model" + path: "/tmp/dummy_model.xml" + precision: "FP32" + +# Run configuration +run: + repeats: 1 + warmup: 0 + cooldown_sec: 0 + matrix: + niter: [10] + nstreams: ["1"] + device: ["CPU"] + +# Reporting +report: + sinks: + - type: "csv" + path: "experiments/results/ssh_test.csv" + - type: "json" + path: "experiments/results/ssh_test.json" + tags: + test_type: "ssh_localhost" + ci: false \ No newline at end of file diff --git a/ovmobilebench/cli.py b/ovmobilebench/cli.py index a79c31e..c93796d 100644 --- a/ovmobilebench/cli.py +++ b/ovmobilebench/cli.py @@ -152,5 +152,27 @@ def list_devices(): console.print(f" • {serial} [[{status_color}]{status}[/{status_color}]]") +@app.command("list-ssh-devices") +def list_ssh_devices(): + """List available SSH devices.""" + from .devices.linux_ssh import list_ssh_devices as list_ssh + from rich.console import Console + + console = Console() + devices = list_ssh() + + if not devices: + console.print("[yellow]No SSH devices configured[/yellow]") + console.print("\nTo configure SSH devices, add them to your experiment YAML") + return + + console.print("[bold green]Available SSH devices:[/bold green]") + for device in devices: + status_color = "green" if device.get("status") == "available" else "yellow" + serial = device.get("serial", "unknown") + status = device.get("status", "unknown") + console.print(f" • {serial} [[{status_color}]{status}[/{status_color}]]") + + if __name__ == "__main__": app() diff --git a/ovmobilebench/config/schema.py b/ovmobilebench/config/schema.py index d9ffe07..5e0dc19 100644 --- a/ovmobilebench/config/schema.py +++ b/ovmobilebench/config/schema.py @@ -49,17 +49,43 @@ class DeviceConfig(BaseModel): """Device configuration.""" kind: Literal["android", "linux_ssh", "ios"] = Field("android", description="Device type") + type: Optional[Literal["android", "linux_ssh", "ios"]] = Field(None, description="Alternative type field") serials: List[str] = Field(default_factory=list, description="Device serials (Android)") host: Optional[str] = Field(None, description="SSH host (Linux)") - user: Optional[str] = Field(None, description="SSH user (Linux)") - key_path: Optional[str] = Field(None, description="SSH key path (Linux)") + username: Optional[str] = Field(None, description="SSH username (Linux)") + user: Optional[str] = Field(None, description="SSH user (Linux) - deprecated, use username") + password: Optional[str] = Field(None, description="SSH password (Linux)") + key_filename: Optional[str] = Field(None, description="SSH key file path (Linux)") + key_path: Optional[str] = Field(None, description="SSH key path (Linux) - deprecated, use key_filename") + port: Optional[int] = Field(22, description="SSH port (Linux)") push_dir: str = Field(default="/data/local/tmp/ovmobilebench", description="Remote directory") use_root: bool = Field(default=False, description="Use root access") @model_validator(mode="after") def validate_device(self): + # Support both 'kind' and 'type' fields + if self.type and not self.kind: + self.kind = self.type + elif self.kind and not self.type: + self.type = self.kind + + # Support deprecated field names + if self.user and not self.username: + self.username = self.user + if self.key_path and not self.key_filename: + self.key_filename = self.key_path + + # Validate based on device type if self.kind == "android" and not self.serials: - raise ValueError("Android device requires at least one serial") + # For Android, allow empty serials (will auto-detect) + pass + elif self.kind == "linux_ssh" or self.type == "linux_ssh": + # For SSH, create a dummy serial if not provided + if not self.serials: + if self.host and self.username: + self.serials = [f"{self.username}@{self.host}:{self.port}"] + elif self.host: + self.serials = [f"{self.host}:{self.port}"] return self diff --git a/ovmobilebench/devices/base.py b/ovmobilebench/devices/base.py index 443e778..463dc7d 100644 --- a/ovmobilebench/devices/base.py +++ b/ovmobilebench/devices/base.py @@ -59,3 +59,7 @@ def cleanup(self, remote_path: str) -> None: """Clean up temporary files on device.""" if self.exists(remote_path): self.rm(remote_path, recursive=True) + + def get_env(self) -> Dict[str, str]: + """Get environment variables for benchmark execution.""" + return {} diff --git a/ovmobilebench/devices/linux_ssh.py b/ovmobilebench/devices/linux_ssh.py new file mode 100644 index 0000000..9c9f644 --- /dev/null +++ b/ovmobilebench/devices/linux_ssh.py @@ -0,0 +1,298 @@ +"""Linux SSH device implementation.""" + +from pathlib import Path +from typing import Dict, Any, Optional, List +import paramiko +import os +from .base import Device +from ..core.errors import DeviceError +from ..core.logging import get_logger + +logger = get_logger(__name__) + + +class LinuxSSHDevice(Device): + """Linux device accessed via SSH.""" + + def __init__( + self, + host: str, + username: str, + password: Optional[str] = None, + key_filename: Optional[str] = None, + port: int = 22, + push_dir: str = "/tmp/ovmobilebench", + ): + """Initialize SSH device. + + Args: + host: Hostname or IP address + username: SSH username + password: SSH password (optional if using key) + key_filename: Path to private key file (optional) + port: SSH port (default 22) + push_dir: Remote directory for deployment + """ + super().__init__(f"{username}@{host}:{port}") + self.serial = f"{username}@{host}:{port}" + self.host = host + self.username = username + self.password = password + self.key_filename = key_filename + self.port = port + self.push_dir = push_dir + self.client: Optional[paramiko.SSHClient] = None + self.sftp: Optional[paramiko.SFTPClient] = None + self._connect() + + def _connect(self): + """Establish SSH connection.""" + try: + self.client = paramiko.SSHClient() + self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + # Prepare connection kwargs + connect_kwargs = { + "hostname": self.host, + "port": self.port, + "username": self.username, + } + + if self.key_filename: + # Expand ~ in key path + key_path = os.path.expanduser(self.key_filename) + if os.path.exists(key_path): + connect_kwargs["key_filename"] = key_path + else: + logger.warning(f"Key file not found: {key_path}, trying password auth") + if self.password: + connect_kwargs["password"] = self.password + elif self.password: + connect_kwargs["password"] = self.password + else: + # Try to use SSH agent or default keys + connect_kwargs["look_for_keys"] = True + connect_kwargs["allow_agent"] = True + + self.client.connect(**connect_kwargs) + self.sftp = self.client.open_sftp() + logger.info(f"Connected to {self.serial}") + except Exception as e: + raise DeviceError(f"Failed to connect to {self.serial}: {e}") + + def push(self, local: Path, remote: str) -> None: + """Push file to device via SFTP.""" + if not self.sftp: + raise DeviceError("SFTP connection not established") + + try: + # Create remote directory if needed + remote_dir = str(Path(remote).parent) + self._mkdir_p(remote_dir) + + # Upload file + self.sftp.put(str(local), remote) + logger.debug(f"Pushed {local} to {remote}") + + # Make executable if it's a binary + if local.suffix in ["", ".sh"]: + self.shell(f"chmod +x {remote}") + except Exception as e: + raise DeviceError(f"Failed to push {local}: {e}") + + def pull(self, remote: str, local: Path) -> None: + """Pull file from device via SFTP.""" + if not self.sftp: + raise DeviceError("SFTP connection not established") + + try: + # Create local directory if needed + local.parent.mkdir(parents=True, exist_ok=True) + + # Download file + self.sftp.get(remote, str(local)) + logger.debug(f"Pulled {remote} to {local}") + except Exception as e: + raise DeviceError(f"Failed to pull {remote}: {e}") + + def shell(self, cmd: str, timeout: int = 120) -> tuple[int, str, str]: + """Execute command on device via SSH.""" + if not self.client: + raise DeviceError("SSH connection not established") + + try: + # Execute command + stdin, stdout, stderr = self.client.exec_command(cmd, timeout=timeout) + + # Read output + stdout_text = stdout.read().decode("utf-8", errors="replace") + stderr_text = stderr.read().decode("utf-8", errors="replace") + returncode = stdout.channel.recv_exit_status() + + logger.debug(f"Command: {cmd}") + logger.debug(f"Return code: {returncode}") + + return returncode, stdout_text, stderr_text + except Exception as e: + raise DeviceError(f"Failed to execute command: {e}") + + def exists(self, remote_path: str) -> bool: + """Check if file/directory exists on device.""" + try: + if self.sftp: + self.sftp.stat(remote_path) + return True + except FileNotFoundError: + pass + except Exception as e: + logger.warning(f"Error checking {remote_path}: {e}") + return False + + def mkdir(self, path: str) -> None: + """Create directory on device.""" + self._mkdir_p(path) + + def _mkdir_p(self, path: str) -> None: + """Create directory recursively (like mkdir -p).""" + if not self.sftp: + raise DeviceError("SFTP connection not established") + + try: + # Check if already exists + self.sftp.stat(path) + return + except FileNotFoundError: + # Create parent first + parent = str(Path(path).parent) + if parent != "/" and parent != ".": + self._mkdir_p(parent) + + # Create this directory + try: + self.sftp.mkdir(path) + logger.debug(f"Created directory: {path}") + except Exception: + # May already exist due to race condition + pass + + def rm(self, path: str, recursive: bool = False) -> None: + """Remove file or directory from device.""" + if recursive: + cmd = f"rm -rf {path}" + else: + cmd = f"rm -f {path}" + + returncode, _, stderr = self.shell(cmd) + if returncode != 0 and stderr: + logger.warning(f"Failed to remove {path}: {stderr}") + + def info(self) -> Dict[str, Any]: + """Get device information.""" + info = { + "type": "linux_ssh", + "host": self.host, + "port": self.port, + "username": self.username, + } + + # Get system info + try: + # OS info + ret, stdout, _ = self.shell("uname -a") + if ret == 0: + info["kernel"] = stdout.strip() + + # CPU info + ret, stdout, _ = self.shell("nproc") + if ret == 0: + info["cpu_cores"] = int(stdout.strip()) + + # Memory info + ret, stdout, _ = self.shell("free -h | grep Mem | awk '{print $2}'") + if ret == 0: + info["memory"] = stdout.strip() + + # Architecture + ret, stdout, _ = self.shell("uname -m") + if ret == 0: + info["arch"] = stdout.strip() + + # Hostname + ret, stdout, _ = self.shell("hostname") + if ret == 0: + info["hostname"] = stdout.strip() + except Exception as e: + logger.warning(f"Failed to get device info: {e}") + + return info + + def is_available(self) -> bool: + """Check if device is available.""" + try: + if self.client and self.client.get_transport(): + return self.client.get_transport().is_active() + except Exception: + pass + return False + + def get_env(self) -> Dict[str, str]: + """Get environment variables for benchmark execution.""" + env = super().get_env() + + # Add LD_LIBRARY_PATH for shared libraries + lib_path = f"{self.push_dir}/lib" + env["LD_LIBRARY_PATH"] = f"{lib_path}:$LD_LIBRARY_PATH" + + return env + + def __del__(self): + """Clean up SSH connection.""" + try: + if self.sftp: + self.sftp.close() + if self.client: + self.client.close() + except Exception: + pass + + +def list_ssh_devices(config_file: Optional[str] = None) -> List[Dict[str, Any]]: + """List configured SSH devices. + + Args: + config_file: Optional path to SSH hosts config file + + Returns: + List of SSH device info dictionaries + """ + devices = [] + + # Try to connect to localhost as a test + try: + import socket + hostname = socket.gethostname() + username = os.environ.get("USER", "user") + + devices.append({ + "serial": f"{username}@localhost:22", + "host": "localhost", + "port": 22, + "username": username, + "status": "available", + "type": "linux_ssh" + }) + + # Also add actual hostname + if hostname != "localhost": + devices.append({ + "serial": f"{username}@{hostname}:22", + "host": hostname, + "port": 22, + "username": username, + "status": "available", + "type": "linux_ssh" + }) + except Exception as e: + logger.warning(f"Failed to detect SSH devices: {e}") + + return devices \ No newline at end of file diff --git a/ovmobilebench/pipeline.py b/ovmobilebench/pipeline.py index f1c310e..83e65cb 100644 --- a/ovmobilebench/pipeline.py +++ b/ovmobilebench/pipeline.py @@ -1,6 +1,7 @@ """Main orchestration pipeline.""" import logging +import os from pathlib import Path from typing import Optional, List, Dict, Any, Union @@ -192,12 +193,25 @@ def report(self) -> None: sink.write(aggregated, path) logger.info(f"Report written to: {path}") - def _get_device(self, serial: str) -> AndroidDevice: + def _get_device(self, serial: str): """Get device instance.""" if self.config.device.kind == "android": + from .devices.android import AndroidDevice return AndroidDevice(serial, self.config.device.push_dir) + elif self.config.device.type == "linux_ssh": + from .devices.linux_ssh import LinuxSSHDevice + # Parse SSH config from device section + device_config = self.config.device.model_dump() + return LinuxSSHDevice( + host=device_config.get("host", "localhost"), + username=device_config.get("username", os.environ.get("USER", "user")), + password=device_config.get("password"), + key_filename=device_config.get("key_filename"), + port=device_config.get("port", 22), + push_dir=device_config.get("push_dir", "/tmp/ovmobilebench") + ) else: - raise OVMobileBenchError(f"Unsupported device kind: {self.config.device.kind}") + raise OVMobileBenchError(f"Unsupported device kind/type: {self.config.device.kind}/{getattr(self.config.device, 'type', 'unknown')}") def _prepare_device(self, device: AndroidDevice) -> None: """Prepare device for benchmarking.""" diff --git a/tests/test_ssh_device.py b/tests/test_ssh_device.py new file mode 100644 index 0000000..3f8dcf0 --- /dev/null +++ b/tests/test_ssh_device.py @@ -0,0 +1,159 @@ +"""Tests for LinuxSSHDevice.""" + +import pytest +from pathlib import Path +from unittest.mock import Mock, patch, MagicMock +from ovmobilebench.devices.linux_ssh import LinuxSSHDevice, list_ssh_devices +from ovmobilebench.core.errors import DeviceError + + +class TestLinuxSSHDevice: + """Test LinuxSSHDevice functionality.""" + + @patch("ovmobilebench.devices.linux_ssh.paramiko.SSHClient") + def test_device_connection(self, mock_ssh_client): + """Test SSH device connection.""" + # Setup mock + mock_client = Mock() + mock_ssh_client.return_value = mock_client + mock_client.connect.return_value = None + mock_client.open_sftp.return_value = Mock() + + # Create device + device = LinuxSSHDevice( + host="localhost", + username="test", + password="test123", + push_dir="/tmp/test" + ) + + # Verify connection was attempted + mock_client.connect.assert_called_once() + assert device.serial == "test@localhost:22" + assert device.push_dir == "/tmp/test" + + @patch("ovmobilebench.devices.linux_ssh.paramiko.SSHClient") + def test_push_file(self, mock_ssh_client): + """Test pushing file via SFTP.""" + # Setup mock + mock_client = Mock() + mock_sftp = Mock() + mock_ssh_client.return_value = mock_client + mock_client.open_sftp.return_value = mock_sftp + + # Mock exec_command for chmod + mock_stdin = Mock() + mock_stdout = Mock() + mock_stderr = Mock() + mock_stdout.read.return_value = b"" + mock_stderr.read.return_value = b"" + mock_stdout.channel.recv_exit_status.return_value = 0 + mock_client.exec_command.return_value = (mock_stdin, mock_stdout, mock_stderr) + + # Create device and push file + device = LinuxSSHDevice(host="localhost", username="test") + device.push(Path("/tmp/test.txt"), "/remote/test.txt") + + # Verify SFTP put was called + mock_sftp.put.assert_called_once_with("/tmp/test.txt", "/remote/test.txt") + + @patch("ovmobilebench.devices.linux_ssh.paramiko.SSHClient") + def test_shell_command(self, mock_ssh_client): + """Test executing shell command.""" + # Setup mock + mock_client = Mock() + mock_ssh_client.return_value = mock_client + mock_client.open_sftp.return_value = Mock() + + # Mock exec_command + mock_stdin = Mock() + mock_stdout = Mock() + mock_stderr = Mock() + mock_stdout.read.return_value = b"command output" + mock_stderr.read.return_value = b"" + mock_stdout.channel.recv_exit_status.return_value = 0 + mock_client.exec_command.return_value = (mock_stdin, mock_stdout, mock_stderr) + + # Create device and run command + device = LinuxSSHDevice(host="localhost", username="test") + ret, out, err = device.shell("echo test") + + # Verify command execution + mock_client.exec_command.assert_called_with("echo test", timeout=120) + assert ret == 0 + assert out == "command output" + assert err == "" + + @patch("ovmobilebench.devices.linux_ssh.paramiko.SSHClient") + def test_device_info(self, mock_ssh_client): + """Test getting device info.""" + # Setup mock + mock_client = Mock() + mock_ssh_client.return_value = mock_client + mock_client.open_sftp.return_value = Mock() + + # Mock multiple exec_command calls + responses = [ + (0, "Linux localhost 5.15.0", ""), # uname -a + (0, "8", ""), # nproc + (0, "16G", ""), # free -h + (0, "x86_64", ""), # uname -m + (0, "test-host", ""), # hostname + ] + + response_iter = iter(responses) + + def exec_side_effect(cmd, timeout=120): + ret, out, err = next(response_iter, (1, "", "error")) + mock_stdin = Mock() + mock_stdout = Mock() + mock_stderr = Mock() + mock_stdout.read.return_value = out.encode() + mock_stderr.read.return_value = err.encode() + mock_stdout.channel.recv_exit_status.return_value = ret + return mock_stdin, mock_stdout, mock_stderr + + mock_client.exec_command.side_effect = exec_side_effect + + # Create device and get info + device = LinuxSSHDevice(host="localhost", username="test") + info = device.info() + + # Verify info + assert info["type"] == "linux_ssh" + assert info["host"] == "localhost" + assert info["username"] == "test" + assert "kernel" in info + assert "cpu_cores" in info + + @patch("ovmobilebench.devices.linux_ssh.paramiko.SSHClient") + def test_is_available(self, mock_ssh_client): + """Test checking device availability.""" + # Setup mock + mock_client = Mock() + mock_transport = Mock() + mock_ssh_client.return_value = mock_client + mock_client.open_sftp.return_value = Mock() + mock_client.get_transport.return_value = mock_transport + mock_transport.is_active.return_value = True + + # Create device and check availability + device = LinuxSSHDevice(host="localhost", username="test") + assert device.is_available() is True + + # Test when not available + mock_transport.is_active.return_value = False + assert device.is_available() is False + + def test_list_ssh_devices(self): + """Test listing SSH devices.""" + devices = list_ssh_devices() + + # Should detect localhost + assert len(devices) > 0 + + # Check first device + first = devices[0] + assert "localhost" in first["serial"] + assert first["type"] == "linux_ssh" + assert first["status"] == "available" \ No newline at end of file From bf12ac0f4117ab0b39e7b35164ca3a5ccd0ead98 Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Sat, 16 Aug 2025 01:36:07 +0200 Subject: [PATCH 2/5] refactor SSH device handling: format code, update methods, and improve test coverage --- ovmobilebench/cli.py | 4 +- ovmobilebench/config/schema.py | 12 ++-- ovmobilebench/devices/base.py | 2 +- ovmobilebench/devices/linux_ssh.py | 105 +++++++++++++++-------------- ovmobilebench/pipeline.py | 8 ++- tests/test_config.py | 8 +-- tests/test_ssh_device.py | 47 ++++++------- 7 files changed, 98 insertions(+), 88 deletions(-) diff --git a/ovmobilebench/cli.py b/ovmobilebench/cli.py index c93796d..edc4edf 100644 --- a/ovmobilebench/cli.py +++ b/ovmobilebench/cli.py @@ -160,12 +160,12 @@ def list_ssh_devices(): console = Console() devices = list_ssh() - + if not devices: console.print("[yellow]No SSH devices configured[/yellow]") console.print("\nTo configure SSH devices, add them to your experiment YAML") return - + console.print("[bold green]Available SSH devices:[/bold green]") for device in devices: status_color = "green" if device.get("status") == "available" else "yellow" diff --git a/ovmobilebench/config/schema.py b/ovmobilebench/config/schema.py index 5e0dc19..f395535 100644 --- a/ovmobilebench/config/schema.py +++ b/ovmobilebench/config/schema.py @@ -49,14 +49,18 @@ class DeviceConfig(BaseModel): """Device configuration.""" kind: Literal["android", "linux_ssh", "ios"] = Field("android", description="Device type") - type: Optional[Literal["android", "linux_ssh", "ios"]] = Field(None, description="Alternative type field") + type: Optional[Literal["android", "linux_ssh", "ios"]] = Field( + None, description="Alternative type field" + ) serials: List[str] = Field(default_factory=list, description="Device serials (Android)") host: Optional[str] = Field(None, description="SSH host (Linux)") username: Optional[str] = Field(None, description="SSH username (Linux)") user: Optional[str] = Field(None, description="SSH user (Linux) - deprecated, use username") password: Optional[str] = Field(None, description="SSH password (Linux)") key_filename: Optional[str] = Field(None, description="SSH key file path (Linux)") - key_path: Optional[str] = Field(None, description="SSH key path (Linux) - deprecated, use key_filename") + key_path: Optional[str] = Field( + None, description="SSH key path (Linux) - deprecated, use key_filename" + ) port: Optional[int] = Field(22, description="SSH port (Linux)") push_dir: str = Field(default="/data/local/tmp/ovmobilebench", description="Remote directory") use_root: bool = Field(default=False, description="Use root access") @@ -68,13 +72,13 @@ def validate_device(self): self.kind = self.type elif self.kind and not self.type: self.type = self.kind - + # Support deprecated field names if self.user and not self.username: self.username = self.user if self.key_path and not self.key_filename: self.key_filename = self.key_path - + # Validate based on device type if self.kind == "android" and not self.serials: # For Android, allow empty serials (will auto-detect) diff --git a/ovmobilebench/devices/base.py b/ovmobilebench/devices/base.py index 463dc7d..bdf4139 100644 --- a/ovmobilebench/devices/base.py +++ b/ovmobilebench/devices/base.py @@ -59,7 +59,7 @@ def cleanup(self, remote_path: str) -> None: """Clean up temporary files on device.""" if self.exists(remote_path): self.rm(remote_path, recursive=True) - + def get_env(self) -> Dict[str, str]: """Get environment variables for benchmark execution.""" return {} diff --git a/ovmobilebench/devices/linux_ssh.py b/ovmobilebench/devices/linux_ssh.py index 9c9f644..769c0f5 100644 --- a/ovmobilebench/devices/linux_ssh.py +++ b/ovmobilebench/devices/linux_ssh.py @@ -24,7 +24,7 @@ def __init__( push_dir: str = "/tmp/ovmobilebench", ): """Initialize SSH device. - + Args: host: Hostname or IP address username: SSH username @@ -50,14 +50,14 @@ def _connect(self): try: self.client = paramiko.SSHClient() self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - + # Prepare connection kwargs connect_kwargs = { "hostname": self.host, "port": self.port, "username": self.username, } - + if self.key_filename: # Expand ~ in key path key_path = os.path.expanduser(self.key_filename) @@ -73,7 +73,7 @@ def _connect(self): # Try to use SSH agent or default keys connect_kwargs["look_for_keys"] = True connect_kwargs["allow_agent"] = True - + self.client.connect(**connect_kwargs) self.sftp = self.client.open_sftp() logger.info(f"Connected to {self.serial}") @@ -84,16 +84,16 @@ def push(self, local: Path, remote: str) -> None: """Push file to device via SFTP.""" if not self.sftp: raise DeviceError("SFTP connection not established") - + try: # Create remote directory if needed remote_dir = str(Path(remote).parent) self._mkdir_p(remote_dir) - + # Upload file self.sftp.put(str(local), remote) logger.debug(f"Pushed {local} to {remote}") - + # Make executable if it's a binary if local.suffix in ["", ".sh"]: self.shell(f"chmod +x {remote}") @@ -104,34 +104,34 @@ def pull(self, remote: str, local: Path) -> None: """Pull file from device via SFTP.""" if not self.sftp: raise DeviceError("SFTP connection not established") - + try: # Create local directory if needed local.parent.mkdir(parents=True, exist_ok=True) - + # Download file self.sftp.get(remote, str(local)) logger.debug(f"Pulled {remote} to {local}") except Exception as e: raise DeviceError(f"Failed to pull {remote}: {e}") - def shell(self, cmd: str, timeout: int = 120) -> tuple[int, str, str]: + def shell(self, cmd: str, timeout: Optional[int] = 120) -> tuple[int, str, str]: """Execute command on device via SSH.""" if not self.client: raise DeviceError("SSH connection not established") - + try: # Execute command stdin, stdout, stderr = self.client.exec_command(cmd, timeout=timeout) - + # Read output stdout_text = stdout.read().decode("utf-8", errors="replace") stderr_text = stderr.read().decode("utf-8", errors="replace") returncode = stdout.channel.recv_exit_status() - + logger.debug(f"Command: {cmd}") logger.debug(f"Return code: {returncode}") - + return returncode, stdout_text, stderr_text except Exception as e: raise DeviceError(f"Failed to execute command: {e}") @@ -156,7 +156,7 @@ def _mkdir_p(self, path: str) -> None: """Create directory recursively (like mkdir -p).""" if not self.sftp: raise DeviceError("SFTP connection not established") - + try: # Check if already exists self.sftp.stat(path) @@ -166,7 +166,7 @@ def _mkdir_p(self, path: str) -> None: parent = str(Path(path).parent) if parent != "/" and parent != ".": self._mkdir_p(parent) - + # Create this directory try: self.sftp.mkdir(path) @@ -181,7 +181,7 @@ def rm(self, path: str, recursive: bool = False) -> None: cmd = f"rm -rf {path}" else: cmd = f"rm -f {path}" - + returncode, _, stderr = self.shell(cmd) if returncode != 0 and stderr: logger.warning(f"Failed to remove {path}: {stderr}") @@ -194,43 +194,45 @@ def info(self) -> Dict[str, Any]: "port": self.port, "username": self.username, } - + # Get system info try: # OS info ret, stdout, _ = self.shell("uname -a") if ret == 0: info["kernel"] = stdout.strip() - + # CPU info ret, stdout, _ = self.shell("nproc") if ret == 0: info["cpu_cores"] = int(stdout.strip()) - + # Memory info ret, stdout, _ = self.shell("free -h | grep Mem | awk '{print $2}'") if ret == 0: info["memory"] = stdout.strip() - + # Architecture ret, stdout, _ = self.shell("uname -m") if ret == 0: info["arch"] = stdout.strip() - + # Hostname ret, stdout, _ = self.shell("hostname") if ret == 0: info["hostname"] = stdout.strip() except Exception as e: logger.warning(f"Failed to get device info: {e}") - + return info def is_available(self) -> bool: """Check if device is available.""" try: - if self.client and self.client.get_transport(): - return self.client.get_transport().is_active() + if self.client: + transport = self.client.get_transport() + if transport: + return bool(transport.is_active()) except Exception: pass return False @@ -238,11 +240,11 @@ def is_available(self) -> bool: def get_env(self) -> Dict[str, str]: """Get environment variables for benchmark execution.""" env = super().get_env() - + # Add LD_LIBRARY_PATH for shared libraries lib_path = f"{self.push_dir}/lib" env["LD_LIBRARY_PATH"] = f"{lib_path}:$LD_LIBRARY_PATH" - + return env def __del__(self): @@ -258,41 +260,46 @@ def __del__(self): def list_ssh_devices(config_file: Optional[str] = None) -> List[Dict[str, Any]]: """List configured SSH devices. - + Args: config_file: Optional path to SSH hosts config file - + Returns: List of SSH device info dictionaries """ devices = [] - + # Try to connect to localhost as a test try: import socket + hostname = socket.gethostname() username = os.environ.get("USER", "user") - - devices.append({ - "serial": f"{username}@localhost:22", - "host": "localhost", - "port": 22, - "username": username, - "status": "available", - "type": "linux_ssh" - }) - - # Also add actual hostname - if hostname != "localhost": - devices.append({ - "serial": f"{username}@{hostname}:22", - "host": hostname, + + devices.append( + { + "serial": f"{username}@localhost:22", + "host": "localhost", "port": 22, "username": username, "status": "available", - "type": "linux_ssh" - }) + "type": "linux_ssh", + } + ) + + # Also add actual hostname + if hostname != "localhost": + devices.append( + { + "serial": f"{username}@{hostname}:22", + "host": hostname, + "port": 22, + "username": username, + "status": "available", + "type": "linux_ssh", + } + ) except Exception as e: logger.warning(f"Failed to detect SSH devices: {e}") - - return devices \ No newline at end of file + + return devices diff --git a/ovmobilebench/pipeline.py b/ovmobilebench/pipeline.py index 83e65cb..cfe5c0c 100644 --- a/ovmobilebench/pipeline.py +++ b/ovmobilebench/pipeline.py @@ -197,9 +197,11 @@ def _get_device(self, serial: str): """Get device instance.""" if self.config.device.kind == "android": from .devices.android import AndroidDevice + return AndroidDevice(serial, self.config.device.push_dir) elif self.config.device.type == "linux_ssh": from .devices.linux_ssh import LinuxSSHDevice + # Parse SSH config from device section device_config = self.config.device.model_dump() return LinuxSSHDevice( @@ -208,10 +210,12 @@ def _get_device(self, serial: str): password=device_config.get("password"), key_filename=device_config.get("key_filename"), port=device_config.get("port", 22), - push_dir=device_config.get("push_dir", "/tmp/ovmobilebench") + push_dir=device_config.get("push_dir", "/tmp/ovmobilebench"), ) else: - raise OVMobileBenchError(f"Unsupported device kind/type: {self.config.device.kind}/{getattr(self.config.device, 'type', 'unknown')}") + raise OVMobileBenchError( + f"Unsupported device kind/type: {self.config.device.kind}/{getattr(self.config.device, 'type', 'unknown')}" + ) def _prepare_device(self, device: AndroidDevice) -> None: """Prepare device for benchmarking.""" diff --git a/tests/test_config.py b/tests/test_config.py index ab5206b..858d30e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -45,10 +45,10 @@ def test_android_device_valid(self): assert device.serials == ["12345678"] def test_android_device_no_serial(self): - """Test Android device without serial.""" - with pytest.raises(ValidationError) as exc_info: - DeviceConfig(kind="android", serials=[]) # Empty serials for Android - assert "Android device requires at least one serial" in str(exc_info.value) + """Test Android device without serial - should be allowed for auto-detect.""" + device = DeviceConfig(kind="android", serials=[]) + assert device.kind == "android" + assert device.serials == [] # Empty serials allowed for auto-detect def test_linux_ssh_device(self): """Test Linux SSH device configuration.""" diff --git a/tests/test_ssh_device.py b/tests/test_ssh_device.py index 3f8dcf0..8d60a9b 100644 --- a/tests/test_ssh_device.py +++ b/tests/test_ssh_device.py @@ -1,10 +1,8 @@ """Tests for LinuxSSHDevice.""" -import pytest from pathlib import Path -from unittest.mock import Mock, patch, MagicMock +from unittest.mock import Mock, patch from ovmobilebench.devices.linux_ssh import LinuxSSHDevice, list_ssh_devices -from ovmobilebench.core.errors import DeviceError class TestLinuxSSHDevice: @@ -18,15 +16,12 @@ def test_device_connection(self, mock_ssh_client): mock_ssh_client.return_value = mock_client mock_client.connect.return_value = None mock_client.open_sftp.return_value = Mock() - + # Create device device = LinuxSSHDevice( - host="localhost", - username="test", - password="test123", - push_dir="/tmp/test" + host="localhost", username="test", password="test123", push_dir="/tmp/test" ) - + # Verify connection was attempted mock_client.connect.assert_called_once() assert device.serial == "test@localhost:22" @@ -40,7 +35,7 @@ def test_push_file(self, mock_ssh_client): mock_sftp = Mock() mock_ssh_client.return_value = mock_client mock_client.open_sftp.return_value = mock_sftp - + # Mock exec_command for chmod mock_stdin = Mock() mock_stdout = Mock() @@ -49,11 +44,11 @@ def test_push_file(self, mock_ssh_client): mock_stderr.read.return_value = b"" mock_stdout.channel.recv_exit_status.return_value = 0 mock_client.exec_command.return_value = (mock_stdin, mock_stdout, mock_stderr) - + # Create device and push file device = LinuxSSHDevice(host="localhost", username="test") device.push(Path("/tmp/test.txt"), "/remote/test.txt") - + # Verify SFTP put was called mock_sftp.put.assert_called_once_with("/tmp/test.txt", "/remote/test.txt") @@ -64,7 +59,7 @@ def test_shell_command(self, mock_ssh_client): mock_client = Mock() mock_ssh_client.return_value = mock_client mock_client.open_sftp.return_value = Mock() - + # Mock exec_command mock_stdin = Mock() mock_stdout = Mock() @@ -73,11 +68,11 @@ def test_shell_command(self, mock_ssh_client): mock_stderr.read.return_value = b"" mock_stdout.channel.recv_exit_status.return_value = 0 mock_client.exec_command.return_value = (mock_stdin, mock_stdout, mock_stderr) - + # Create device and run command device = LinuxSSHDevice(host="localhost", username="test") ret, out, err = device.shell("echo test") - + # Verify command execution mock_client.exec_command.assert_called_with("echo test", timeout=120) assert ret == 0 @@ -91,7 +86,7 @@ def test_device_info(self, mock_ssh_client): mock_client = Mock() mock_ssh_client.return_value = mock_client mock_client.open_sftp.return_value = Mock() - + # Mock multiple exec_command calls responses = [ (0, "Linux localhost 5.15.0", ""), # uname -a @@ -100,9 +95,9 @@ def test_device_info(self, mock_ssh_client): (0, "x86_64", ""), # uname -m (0, "test-host", ""), # hostname ] - + response_iter = iter(responses) - + def exec_side_effect(cmd, timeout=120): ret, out, err = next(response_iter, (1, "", "error")) mock_stdin = Mock() @@ -112,13 +107,13 @@ def exec_side_effect(cmd, timeout=120): mock_stderr.read.return_value = err.encode() mock_stdout.channel.recv_exit_status.return_value = ret return mock_stdin, mock_stdout, mock_stderr - + mock_client.exec_command.side_effect = exec_side_effect - + # Create device and get info device = LinuxSSHDevice(host="localhost", username="test") info = device.info() - + # Verify info assert info["type"] == "linux_ssh" assert info["host"] == "localhost" @@ -136,11 +131,11 @@ def test_is_available(self, mock_ssh_client): mock_client.open_sftp.return_value = Mock() mock_client.get_transport.return_value = mock_transport mock_transport.is_active.return_value = True - + # Create device and check availability device = LinuxSSHDevice(host="localhost", username="test") assert device.is_available() is True - + # Test when not available mock_transport.is_active.return_value = False assert device.is_available() is False @@ -148,12 +143,12 @@ def test_is_available(self, mock_ssh_client): def test_list_ssh_devices(self): """Test listing SSH devices.""" devices = list_ssh_devices() - + # Should detect localhost assert len(devices) > 0 - + # Check first device first = devices[0] assert "localhost" in first["serial"] assert first["type"] == "linux_ssh" - assert first["status"] == "available" \ No newline at end of file + assert first["status"] == "available" From de0a85bf6effccde8422cc60905b41305b5d13bd Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Sat, 16 Aug 2025 01:38:44 +0200 Subject: [PATCH 3/5] add `types-paramiko` to dependencies in `requirements.txt` --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0b68af1..c4cefec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ pytest-cov>=5.0.0 mypy>=1.10.0 ruff>=0.5.0 black>=24.4.2 -types-PyYAML>=6.0.12 \ No newline at end of file +types-PyYAML>=6.0.12 +types-paramiko>=3.4.0 \ No newline at end of file From 0f2cdf8a91aa2f9976657b47e2c8ddfd7b573c29 Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Sat, 16 Aug 2025 01:40:22 +0200 Subject: [PATCH 4/5] remove unused condition from `device-test-ssh` step in CI workflow --- .github/workflows/bench.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index b4a2fab..7f11c4c 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -147,7 +147,6 @@ jobs: # Test SSH connection to localhost device-test-ssh: - if: github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[device-test-ssh]') needs: build-package runs-on: ubuntu-latest steps: From 9a340514e56a5494bc32547358246c9d9ab343c1 Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Sat, 16 Aug 2025 01:52:41 +0200 Subject: [PATCH 5/5] automate SSH setup and testing in CI: add scripts for config generation, test execution, and setup steps; update workflow accordingly --- .github/workflows/bench.yml | 101 ++----------- .gitignore | 6 + scripts/generate_ssh_config.py | 269 +++++++++++++++++++++++++++++++++ 3 files changed, 290 insertions(+), 86 deletions(-) create mode 100755 scripts/generate_ssh_config.py diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 7f11c4c..936b53b 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -110,7 +110,9 @@ jobs: # Optional: Run on a self-hosted runner with a real device device-test-adb: if: github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[device-test-adb]') - needs: build-package + needs: + - build-package + - dry-run runs-on: self-hosted steps: - uses: actions/checkout@v4 @@ -147,7 +149,9 @@ jobs: # Test SSH connection to localhost device-test-ssh: - needs: build-package + needs: + - build-package + - dry-run runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -165,28 +169,9 @@ jobs: - name: Set up SSH server run: | - # Install SSH server if not present - sudo apt-get update - sudo apt-get install -y openssh-server - - # Configure SSH for passwordless localhost access - ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa || true - cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys - chmod 600 ~/.ssh/authorized_keys - - # Configure SSH client - cat > ~/.ssh/config << EOF - Host localhost - StrictHostKeyChecking no - UserKnownHostsFile=/dev/null - EOF - chmod 600 ~/.ssh/config - - # Start SSH service - sudo service ssh start - - # Test connection - ssh -o ConnectTimeout=5 localhost "echo 'SSH connection successful'" + # Generate and run SSH setup script + python scripts/generate_ssh_config.py --type setup + bash scripts/setup_ssh_ci.sh - name: List SSH devices run: | @@ -194,73 +179,17 @@ jobs: - name: Test SSH deployment run: | - # Create test file - echo "test content" > /tmp/test_file.txt - - # Test using Python - python << 'EOF' - from ovmobilebench.devices.linux_ssh import LinuxSSHDevice - import os - from pathlib import Path - - # Connect to localhost - device = LinuxSSHDevice( - host="localhost", - username=os.environ["USER"], - key_filename="~/.ssh/id_rsa", - push_dir="/tmp/ovmobilebench_test" - ) - - # Test operations - print(f"Device available: {device.is_available()}") - print(f"Device info: {device.info()}") - - # Test push - device.push(Path("/tmp/test_file.txt"), "/tmp/ovmobilebench_test/test.txt") - - # Test shell command - ret, out, err = device.shell("cat /tmp/ovmobilebench_test/test.txt") - print(f"File content: {out.strip()}") - - # Test exists - print(f"File exists: {device.exists('/tmp/ovmobilebench_test/test.txt')}") - - # Cleanup - device.rm("/tmp/ovmobilebench_test", recursive=True) - EOF + # Generate and run SSH test script + python scripts/generate_ssh_config.py --type test + python scripts/test_ssh_device_ci.py - name: Run benchmark dry-run via SSH run: | - # Create minimal config for SSH - cat > experiments/ssh_localhost.yaml << 'EOF' - project: - name: "ssh-test" - run_id: "ci-test" - - device: - type: "linux_ssh" - host: "localhost" - username: "${USER}" - key_filename: "~/.ssh/id_rsa" - push_dir: "/tmp/ovmobilebench" - - models: - - name: "dummy" - path: "/tmp/dummy_model.xml" - - run: - repeats: 1 - matrix: - niter: [10] - - report: - sinks: - - type: "csv" - path: "experiments/results/ssh_test.csv" - EOF + # Generate SSH config using Python script + python scripts/generate_ssh_config.py --type config # Run in dry-run mode - ovmobilebench all -c experiments/ssh_localhost.yaml --dry-run || true + ovmobilebench all -c experiments/ssh_localhost_ci.yaml --dry-run || true - name: Upload SSH test results if: always() diff --git a/.gitignore b/.gitignore index aee02c2..5e108d7 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,9 @@ dmypy.json .idea .claude CLAUDE.md + +# Generated CI configs +experiments/ssh_localhost_ci.yaml +experiments/results/ +scripts/test_ssh_device_ci.py +scripts/setup_ssh_ci.sh diff --git a/scripts/generate_ssh_config.py b/scripts/generate_ssh_config.py new file mode 100755 index 0000000..45887de --- /dev/null +++ b/scripts/generate_ssh_config.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Generate SSH configuration and test scripts for CI.""" + +import os +import sys +import yaml +from datetime import datetime +from pathlib import Path +import argparse + + +def generate_ssh_config(output_file: str = "experiments/ssh_localhost_ci.yaml"): + """Generate SSH configuration for CI testing.""" + + # Get current user + username = os.environ.get("USER", "runner") + + # Generate run ID with timestamp + run_id = f"ci-test-{datetime.now().strftime('%Y%m%d-%H%M%S')}" + + config = { + "project": { + "name": "ssh-test", + "run_id": run_id, + "description": "SSH localhost test for CI" + }, + "device": { + "type": "linux_ssh", + "host": "localhost", + "username": username, + "push_dir": "/tmp/ovmobilebench" + }, + "build": { + "enabled": False, + "openvino_repo": "/tmp/openvino" # Dummy path, not used when disabled + }, + "models": [ + { + "name": "dummy", + "path": "/tmp/dummy_model.xml", + "precision": "FP32" + } + ], + "run": { + "repeats": 1, + "warmup": False, + "cooldown_sec": 0, + "matrix": { + "niter": [10], + "device": ["CPU"], + "nstreams": ["1"], + "api": ["sync"], + "nireq": [1], + "infer_precision": ["FP16"], + "threads": [4] + } + }, + "report": { + "sinks": [ + { + "type": "csv", + "path": "experiments/results/ssh_test.csv" + }, + { + "type": "json", + "path": "experiments/results/ssh_test.json" + } + ], + "aggregate": True, + "tags": { + "test_type": "ssh_localhost", + "ci": True, + "user": username + } + } + } + + # Check if SSH key exists + ssh_key_path = Path.home() / ".ssh" / "id_rsa" + if ssh_key_path.exists(): + config["device"]["key_filename"] = str(ssh_key_path) + + # Create output directory if needed + output_path = Path(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Write configuration + with open(output_path, 'w') as f: + yaml.dump(config, f, default_flow_style=False, sort_keys=False) + + print(f"Generated SSH config: {output_file}") + print(f" Username: {username}") + print(f" Run ID: {run_id}") + if ssh_key_path.exists(): + print(f" SSH Key: {ssh_key_path}") + + return output_file + + +def generate_ssh_test_script(output_file: str = "scripts/test_ssh_device.py"): + """Generate SSH device test script.""" + + username = os.environ.get("USER", "runner") + + script_content = f'''#!/usr/bin/env python3 +"""Test SSH device functionality.""" + +from ovmobilebench.devices.linux_ssh import LinuxSSHDevice +import os +from pathlib import Path + +def test_ssh_device(): + """Test SSH device operations.""" + + # Connect to localhost + device = LinuxSSHDevice( + host="localhost", + username="{username}", + key_filename="~/.ssh/id_rsa", + push_dir="/tmp/ovmobilebench_test" + ) + + # Test operations + print(f"Device available: {{device.is_available()}}") + print(f"Device info: {{device.info()}}") + + # Create test file + test_file = Path("/tmp/test_file.txt") + test_file.write_text("test content from CI") + + # Test push + device.push(test_file, "/tmp/ovmobilebench_test/test.txt") + + # Test shell command + ret, out, err = device.shell("cat /tmp/ovmobilebench_test/test.txt") + print(f"File content: {{out.strip()}}") + assert out.strip() == "test content from CI", "File content mismatch" + + # Test exists + exists = device.exists("/tmp/ovmobilebench_test/test.txt") + print(f"File exists: {{exists}}") + assert exists, "File should exist" + + # Test pull + pulled_file = Path("/tmp/pulled_test.txt") + device.pull("/tmp/ovmobilebench_test/test.txt", pulled_file) + assert pulled_file.read_text() == "test content from CI", "Pulled file content mismatch" + + # Cleanup + device.rm("/tmp/ovmobilebench_test", recursive=True) + test_file.unlink() + pulled_file.unlink() + + print("All SSH tests passed!") + +if __name__ == "__main__": + test_ssh_device() +''' + + # Create output directory if needed + output_path = Path(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Write script + with open(output_path, 'w') as f: + f.write(script_content) + + # Make executable + output_path.chmod(0o755) + + print(f"Generated SSH test script: {output_file}") + return output_file + + +def generate_ssh_setup_script(output_file: str = "scripts/setup_ssh_ci.sh"): + """Generate SSH setup script for CI.""" + + script_content = '''#!/bin/bash +# Setup SSH for CI testing + +set -e + +echo "Setting up SSH server for CI..." + +# Install SSH server if not present +if ! command -v sshd &> /dev/null; then + sudo apt-get update + sudo apt-get install -y openssh-server +fi + +# Generate SSH key if not exists +if [ ! -f ~/.ssh/id_rsa ]; then + ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa +fi + +# Setup authorized keys +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys + +# Configure SSH client +cat > ~/.ssh/config << EOF +Host localhost + StrictHostKeyChecking no + UserKnownHostsFile=/dev/null + LogLevel ERROR +EOF +chmod 600 ~/.ssh/config + +# Start SSH service +sudo service ssh start || sudo systemctl start sshd + +# Wait for SSH to be ready +sleep 2 + +# Test connection +if ssh -o ConnectTimeout=5 localhost "echo 'SSH connection successful'"; then + echo "SSH setup completed successfully" +else + echo "SSH connection test failed" + exit 1 +fi +''' + + # Create output directory if needed + output_path = Path(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Write script + with open(output_path, 'w') as f: + f.write(script_content) + + # Make executable + output_path.chmod(0o755) + + print(f"Generated SSH setup script: {output_file}") + return output_file + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser(description="Generate SSH test configurations and scripts") + parser.add_argument( + "--type", + choices=["config", "test", "setup", "all"], + default="config", + help="Type of file to generate" + ) + parser.add_argument( + "--output", + help="Output file path (optional, uses defaults)" + ) + + args = parser.parse_args() + + if args.type == "config" or args.type == "all": + output = args.output if args.output and args.type == "config" else "experiments/ssh_localhost_ci.yaml" + generate_ssh_config(output) + + if args.type == "test" or args.type == "all": + output = args.output if args.output and args.type == "test" else "scripts/test_ssh_device_ci.py" + generate_ssh_test_script(output) + + if args.type == "setup" or args.type == "all": + output = args.output if args.output and args.type == "setup" else "scripts/setup_ssh_ci.sh" + generate_ssh_setup_script(output) + + +if __name__ == "__main__": + main() \ No newline at end of file