diff --git a/docs/cli-usage.md b/docs/cli-usage.md deleted file mode 100644 index c710f31..0000000 --- a/docs/cli-usage.md +++ /dev/null @@ -1,700 +0,0 @@ -# CLI Usage Guide - -This guide provides comprehensive documentation for the `airoa-lineage-usb-copy` command-line interface. - -## Table of Contents - -- [Installation](#installation) -- [Quick Start](#quick-start) -- [Configuration](#configuration) - - [Configuration File](#configuration-file) - - [Environment Variables](#environment-variables) - - [Priority Order](#priority-order) -- [Commands](#commands) - - [start](#start-command) - - [complete](#complete-command) - - [cancel](#cancel-command) -- [Global Options](#global-options) -- [Output Modes](#output-modes) -- [Exit Codes](#exit-codes) -- [Examples](#examples) - - [Basic Workflow](#basic-workflow) - - [Using Configuration File](#using-configuration-file) - - [Using Environment Variables](#using-environment-variables) - - [Shell Script Integration](#shell-script-integration) - - [Error Handling](#error-handling) -- [Troubleshooting](#troubleshooting) - ---- - -## Installation - -After installing the `airoa-lineage` package: - -```bash -# Using uv (recommended) -uv pip install -e . - -# Or using pip -pip install -e . -``` - -The `airoa-lineage-usb-copy` command will be available in your PATH. - ---- - -## Quick Start - -```bash -# 1. Start a USB copy session -RUN_ID=$(airoa-lineage-usb-copy start \ - --namespace production \ - --robot-id hsr001 \ - --location weblab \ - --repository-hash $(git rev-parse HEAD) \ - --repository-uri https://github.com/AIRoA/airoa-lineage.git \ - --repository-tag v1.0.0 \ - --repository-branch main \ - --nominal-start-time "2025-11-26T00:00:00+00:00" \ - --nominal-end-time "2025-11-26T05:00:00+00:00") - -# 2. Perform your USB copy operation -# (your data copy processing here) - -# 3. Complete the session -airoa-lineage-usb-copy complete --run-id "$RUN_ID" -``` - ---- - -## Configuration - -The CLI supports three configuration sources with the following priority: - -**CLI arguments > Environment variables > Configuration file > Defaults** - -### Configuration File - -Default path: `~/.config/airoa-lineage/config.json` - -You can override this with the `--config` option or by setting `XDG_CONFIG_HOME`. - -**Example config.json:** - -```json -{ - "marquez_url": "http://localhost:9000", - "namespace": "airoa_production", - "facet_prefix": "airoa", - "job_name": "usb-data-copy", - "common_facet": { - "robotId": "hsr001", - "location": "weblab", - "repositoryHash": "df110d5a8e3b9c2f1a7d6e4f5c3a2b1", - "repositoryUri": "https://github.com/AIRoA/airoa-lineage.git", - "repositoryTag": "v1.0.0", - "repositoryBranch": "main" - } -} -``` - -**Configuration Schema:** - -| Field | Type | Required | Default | Description | -|-------|------|----------|---------|-------------| -| `marquez_url` | string | No | `http://localhost:9000` | Marquez server URL | -| `namespace` | string | **Yes** | - | OpenLineage namespace | -| `job_name` | string | No | `usb-data-copy` | Job identifier | -| `facet_prefix` | string | No | `""` (empty) | Prefix for custom facets | -| `common_facet` | object | **Yes** | - | CommonRunFacet fields | - -**CommonRunFacet Fields:** - -| Field | Type | Required | Description | -|-------|------|----------|-------------| -| `robotId` | string | **Yes** | Robot identifier (e.g., `hsr001`) | -| `location` | string | **Yes** | Location identifier (e.g., `weblab`) | -| `repositoryHash` | string | **Yes** | Git commit hash | -| `repositoryUri` | string | **Yes** | Repository URI | -| `repositoryTag` | string | **Yes** | Git tag (e.g., `v1.0.0`) | -| `repositoryBranch` | string | **Yes** | Git branch (e.g., `main`) | - -### Environment Variables - -All configuration values can be set via environment variables: - -```bash -# Top-level configuration -export AIROA_MARQUEZ_URL=http://localhost:9000 -export AIROA_NAMESPACE=airoa_production -export AIROA_FACET_PREFIX=airoa -export AIROA_JOB_NAME=usb-data-copy - -# CommonRunFacet fields -export AIROA_ROBOT_ID=hsr001 -export AIROA_LOCATION=weblab -export AIROA_REPOSITORY_HASH=$(git rev-parse HEAD) -export AIROA_REPOSITORY_URI=https://github.com/AIRoA/airoa-lineage.git -export AIROA_REPOSITORY_TAG=$(git describe --tags) -export AIROA_REPOSITORY_BRANCH=$(git rev-parse --abbrev-ref HEAD) -``` - -**Note:** `MARQUEZ_URL` is also supported for backward compatibility, but `AIROA_MARQUEZ_URL` takes precedence. - -### Priority Order - -When the same value is specified in multiple places: - -1. **CLI arguments** (highest priority) -2. **Environment variables** -3. **Configuration file** -4. **Default values** (lowest priority) - -**Example:** - -```bash -# config.json has: "namespace": "dev" -# Environment has: AIROA_NAMESPACE=staging - -# This uses CLI argument (production) -airoa-lineage-usb-copy start --namespace production - -# This uses environment variable (staging) -airoa-lineage-usb-copy start - -# If no env var, uses config file (dev) -unset AIROA_NAMESPACE -airoa-lineage-usb-copy start -``` - ---- - -## Commands - -### start Command - -Start a USB copy session and emit a START event to Marquez. - -**Synopsis:** - -```bash -airoa-lineage-usb-copy start [OPTIONS] -``` - -**Options:** - -| Option | Type | Required | Description | -|--------|------|----------|-------------| -| `--namespace` | string | Yes* | OpenLineage namespace | -| `--robot-id` | string | Yes* | Robot identifier | -| `--location` | string | Yes* | Location identifier | -| `--repository-hash` | string | Yes* | Git commit hash | -| `--repository-uri` | string | Yes* | Repository URI | -| `--repository-tag` | string | Yes* | Git tag | -| `--repository-branch` | string | Yes* | Git branch | -| `--nominal-start-time` | string | No | Nominal start time (ISO 8601) | -| `--nominal-end-time` | string | No | Nominal end time (ISO 8601) | -| `--job-name` | string | No | Job name (default: `usb-data-copy`) | -| `--marquez-url` | string | No | Marquez server URL | -| `--facet-prefix` | string | No | Facet prefix | -| `--dry-run` | flag | No | Show what would be done without executing | - -\* Required unless provided via config file or environment variables - -**Output:** - -- **Default:** Prints the `run_id` (UUID) to stdout -- **JSON mode:** `{"run_id": "...", "status": "started"}` -- **Verbose mode:** Additional messages to stderr, run_id to stdout -- **Dry-run mode:** Shows configuration without emitting events - -**Examples:** - -```bash -# Basic usage -RUN_ID=$(airoa-lineage-usb-copy start \ - --namespace production \ - --robot-id hsr001 \ - --location weblab \ - --repository-hash $(git rev-parse HEAD) \ - --repository-uri https://github.com/AIRoA/airoa-lineage.git \ - --repository-tag v1.0.0 \ - --repository-branch main) - -# With nominal times (for batch processing) -RUN_ID=$(airoa-lineage-usb-copy start \ - --nominal-start-time "2025-11-26T00:00:00+00:00" \ - --nominal-end-time "2025-11-26T05:00:00+00:00") - -# Dry-run mode (verify configuration) -airoa-lineage-usb-copy start --dry-run - -# JSON output (for programmatic parsing) -airoa-lineage-usb-copy start --json -``` - ---- - -### complete Command - -Complete a USB copy session and emit a COMPLETE event to Marquez. - -**Synopsis:** - -```bash -airoa-lineage-usb-copy complete --run-id RUN_ID [OPTIONS] -``` - -**Options:** - -| Option | Type | Required | Description | -|--------|------|----------|-------------| -| `--run-id` | string | **Yes** | Session run ID (from start command) | -| `--namespace` | string | Yes* | OpenLineage namespace | -| `--job-name` | string | No | Job name | -| `--marquez-url` | string | No | Marquez server URL | -| `--facet-prefix` | string | No | Facet prefix | -| `--robot-id` | string | Yes* | Robot identifier | -| `--location` | string | Yes* | Location identifier | -| `--repository-hash` | string | Yes* | Git commit hash | -| `--repository-uri` | string | Yes* | Repository URI | -| `--repository-tag` | string | Yes* | Git tag | -| `--repository-branch` | string | Yes* | Git branch | - -\* Required unless provided via config file or environment variables - -**Note:** CommonRunFacet values provided during `complete` do not need to match the `start` values. This allows for flexible scenarios (e.g., multi-robot collaboration). However, ensure consistency when tracking single-robot operations. - -**Output:** - -- **Default:** `Session completed successfully` -- **JSON mode:** `{"run_id": "...", "status": "completed"}` -- **Quiet mode:** No output - -**Examples:** - -```bash -# Basic usage (CommonRunFacet from config file or environment variables) -airoa-lineage-usb-copy complete --run-id "$RUN_ID" - -# With namespace override -airoa-lineage-usb-copy complete --run-id "$RUN_ID" --namespace production - -# Override specific CommonRunFacet fields -airoa-lineage-usb-copy complete \ - --run-id "$RUN_ID" \ - --robot-id hsr002 \ - --location lab2 \ - --repository-hash $(git rev-parse HEAD) - -# Specify all arguments (no config file needed) -airoa-lineage-usb-copy complete \ - --run-id "$RUN_ID" \ - --namespace production \ - --robot-id hsr001 \ - --location weblab \ - --repository-hash abc123 \ - --repository-uri https://github.com/AIRoA/airoa-lineage.git \ - --repository-tag v1.0.0 \ - --repository-branch main - -# JSON output -airoa-lineage-usb-copy complete --run-id "$RUN_ID" --json - -# Quiet mode (no output) -airoa-lineage-usb-copy complete --run-id "$RUN_ID" --quiet -``` - ---- - -### cancel Command - -Cancel a USB copy session and emit an ABORT event to Marquez. - -**Synopsis:** - -```bash -airoa-lineage-usb-copy cancel --run-id RUN_ID [OPTIONS] -``` - -**Options:** - -| Option | Type | Required | Description | -|--------|------|----------|-------------| -| `--run-id` | string | **Yes** | Session run ID (from start command) | -| `--namespace` | string | Yes* | OpenLineage namespace | -| `--job-name` | string | No | Job name | -| `--marquez-url` | string | No | Marquez server URL | -| `--facet-prefix` | string | No | Facet prefix | -| `--robot-id` | string | Yes* | Robot identifier | -| `--location` | string | Yes* | Location identifier | -| `--repository-hash` | string | Yes* | Git commit hash | -| `--repository-uri` | string | Yes* | Repository URI | -| `--repository-tag` | string | Yes* | Git tag | -| `--repository-branch` | string | Yes* | Git branch | - -\* Required unless provided via config file or environment variables - -**Note:** CommonRunFacet values provided during `cancel` do not need to match the `start` values. This allows for flexible scenarios (e.g., multi-robot collaboration). However, ensure consistency when tracking single-robot operations. - -**Output:** - -- **Default:** `Session cancelled successfully` -- **JSON mode:** `{"run_id": "...", "status": "cancelled"}` -- **Quiet mode:** No output - -**Examples:** - -```bash -# Basic usage (CommonRunFacet from config file or environment variables) -airoa-lineage-usb-copy cancel --run-id "$RUN_ID" - -# Override specific CommonRunFacet fields -airoa-lineage-usb-copy cancel \ - --run-id "$RUN_ID" \ - --robot-id hsr002 \ - --location lab2 - -# Specify all arguments (no config file needed) -airoa-lineage-usb-copy cancel \ - --run-id "$RUN_ID" \ - --namespace production \ - --robot-id hsr001 \ - --location weblab \ - --repository-hash abc123 \ - --repository-uri https://github.com/AIRoA/airoa-lineage.git \ - --repository-tag v1.0.0 \ - --repository-branch main - -# JSON output -airoa-lineage-usb-copy cancel --run-id "$RUN_ID" --json - -# Quiet mode (no output) -airoa-lineage-usb-copy cancel --run-id "$RUN_ID" --quiet -``` - ---- - -## Global Options - -These options apply to all commands: - -| Option | Short | Description | -|--------|-------|-------------| -| `--config PATH` | - | Path to config file (default: `~/.config/airoa-lineage/config.json`) | -| `--verbose` | `-v` | Enable verbose output (messages to stderr) | -| `--quiet` | `-q` | Suppress output except errors | -| `--json` | - | Output in JSON format | -| `--version` | - | Show version and exit | -| `--help` | `-h` | Show help message and exit | - -**Examples:** - -```bash -# Show version -airoa-lineage-usb-copy --version - -# Show help -airoa-lineage-usb-copy --help -airoa-lineage-usb-copy start --help - -# Use custom config file -airoa-lineage-usb-copy --config /path/to/config.json start - -# Verbose mode -airoa-lineage-usb-copy -v start - -# JSON output -airoa-lineage-usb-copy --json start -``` - ---- - -## Output Modes - -### Default Mode - -- **start:** Prints only `run_id` to stdout (for piping) -- **complete/cancel:** Prints success message to stdout - -```bash -RUN_ID=$(airoa-lineage-usb-copy start) -# Output: 550e8400-e29b-41d4-a716-446655440000 -``` - -### Verbose Mode (`-v`, `--verbose`) - -- All messages go to **stderr** -- `run_id` goes to **stdout** (for piping) -- Includes configuration details and event status - -```bash -airoa-lineage-usb-copy -v start -# stderr: Loading config from ~/.config/airoa-lineage/config.json -# stderr: Connecting to Marquez at http://localhost:9000 -# stderr: Session started successfully -# stdout: 550e8400-e29b-41d4-a716-446655440000 -``` - -### Quiet Mode (`-q`, `--quiet`) - -- **start:** Prints only `run_id` to stdout -- **complete/cancel:** No output (silent) -- Errors still go to stderr - -```bash -airoa-lineage-usb-copy -q complete --run-id "$RUN_ID" -# (no output) -``` - -### JSON Mode (`--json`) - -- All output in JSON format -- Suitable for programmatic parsing - -```bash -airoa-lineage-usb-copy --json start -# {"run_id": "550e8400-e29b-41d4-a716-446655440000", "status": "started"} - -airoa-lineage-usb-copy --json complete --run-id "$RUN_ID" -# {"run_id": "550e8400-e29b-41d4-a716-446655440000", "status": "completed"} -``` - ---- - -## Exit Codes - -The CLI uses standard exit codes to indicate success or failure: - -| Code | Meaning | Description | -|------|---------|-------------| -| `0` | Success | Command completed successfully | -| `1` | General error | Session error, runtime error, or unknown error | -| `2` | Connection error | Failed to connect to Marquez server | -| `3` | Configuration error | Missing or invalid configuration | - -**Example:** - -```bash -airoa-lineage-usb-copy start -EXIT_CODE=$? - -if [ $EXIT_CODE -eq 0 ]; then - echo "Success" -elif [ $EXIT_CODE -eq 3 ]; then - echo "Configuration error - check your config file" -elif [ $EXIT_CODE -eq 2 ]; then - echo "Connection error - is Marquez running?" -else - echo "Unknown error" -fi -``` - ---- - -## Examples - -For complete working examples, see [examples/cli/usb_copy/](../examples/cli/usb_copy/). - -### Available Examples - -The following executable shell scripts demonstrate different usage patterns: - -1. **[basic_workflow.sh](../examples/cli/usb_copy/basic_workflow.sh)** - Simple start → complete workflow - - All configuration via CLI arguments - - Ideal for quick testing and learning - -2. **[with_config_file.sh](../examples/cli/usb_copy/with_config_file.sh)** - Configuration file usage - - Loads settings from `~/.config/airoa-lineage/config.json` - - Minimal CLI arguments required - - Includes dry-run mode for testing - -3. **[with_env_vars.sh](../examples/cli/usb_copy/with_env_vars.sh)** - Environment variables - - Configuration via `AIROA_*` environment variables - - Dynamically retrieves repository info from git - -4. **[error_handling.sh](../examples/cli/usb_copy/error_handling.sh)** - Error handling with trap - - Automatic session cancellation on error - - Cleanup function with proper exit codes - -5. **[retry_logic.sh](../examples/cli/usb_copy/retry_logic.sh)** - Connection retry logic - - Automatic retry on connection errors - - Maximum 3 attempts with exponential backoff - -See [examples/cli/usb_copy/README.md](../examples/cli/usb_copy/README.md) for detailed documentation on each example, including prerequisites, usage instructions, and integration patterns. - ---- - -## Troubleshooting - -### Common Errors - -#### 1. "Missing required CommonRunFacet fields" - -**Error message:** -``` -Configuration error: Missing required CommonRunFacet fields: robotId, location, repositoryHash, repositoryUri, repositoryTag, repositoryBranch. -Please provide them in config file, environment variables, or CLI arguments. -``` - -**Solution:** - -Provide all required fields via one of these methods: - -```bash -# Option 1: CLI arguments -airoa-lineage-usb-copy start \ - --robot-id hsr001 \ - --location weblab \ - --repository-hash $(git rev-parse HEAD) \ - --repository-uri https://github.com/AIRoA/airoa-lineage.git \ - --repository-tag v1.0.0 \ - --repository-branch main - -# Option 2: Environment variables -export AIROA_ROBOT_ID=hsr001 -export AIROA_LOCATION=weblab -export AIROA_REPOSITORY_HASH=$(git rev-parse HEAD) -export AIROA_REPOSITORY_URI=https://github.com/AIRoA/airoa-lineage.git -export AIROA_REPOSITORY_TAG=v1.0.0 -export AIROA_REPOSITORY_BRANCH=main - -# Option 3: Config file -cat > ~/.config/airoa-lineage/config.json < ~/.config/airoa-lineage/config.json -``` - -#### 3. "Invalid run_id format" - -**Error message:** -``` -Configuration error: Invalid run_id format: not-a-uuid. Must be a valid UUID. -``` - -**Solution:** - -The `run_id` must be a valid UUID (36 characters with hyphens). Get the correct run_id from the `start` command: - -```bash -# Correct usage -RUN_ID=$(airoa-lineage-usb-copy start) -airoa-lineage-usb-copy complete --run-id "$RUN_ID" -``` - -#### 4. "Cannot resume session. Session already completed." - -**Error message:** -``` -Session error: Cannot resume session 550e8400-e29b-41d4-a716-446655440000. Session already completed. -``` - -**Cause:** You're trying to complete or cancel a session that was already completed. - -**Solution:** Each session can only be completed or cancelled once. Start a new session if needed. - -#### 5. Connection refused / Marquez server not running - -**Error message:** -``` -Connection error: Failed to connect to Marquez at http://localhost:9000 -``` - -**Solution:** - -Start the Marquez server: - -```bash -# Using Docker -docker run -d -p 3000:3000 -p 9000:9000 marquezproject/marquez:latest - -# Verify it's running -curl http://localhost:9000/api/v1/namespaces -``` - -### Debugging Tips - -#### 1. Use verbose mode - -```bash -airoa-lineage-usb-copy -v start -``` - -This shows: -- Config file path being used -- Loaded configuration values -- Marquez connection details -- Event emission status - -#### 2. Use dry-run mode - -```bash -airoa-lineage-usb-copy start --dry-run -``` - -This shows what would be sent to Marquez without actually sending it. - -#### 3. Check configuration loading - -```bash -# Test with explicit config file -airoa-lineage-usb-copy --config ~/.config/airoa-lineage/config.json start --dry-run - -# Verify environment variables -env | grep AIROA_ -``` - -#### 4. Verify Marquez connection - -```bash -# Test Marquez API -curl http://localhost:9000/api/v1/namespaces - -# Check Marquez logs -docker logs -``` - -#### 5. Check JSON syntax - -```bash -# Validate config file JSON -cat ~/.config/airoa-lineage/config.json | python -m json.tool -``` - -### Getting Help - -- CLI help: `airoa-lineage-usb-copy --help` -- Command help: `airoa-lineage-usb-copy start --help` -- Version: `airoa-lineage-usb-copy --version` -- GitHub Issues: https://github.com/AIRoA/airoa-lineage/issues -- OpenLineage docs: https://openlineage.io/docs -- Marquez docs: https://marquezproject.github.io/marquez/ diff --git a/docs/cli/overview.md b/docs/cli/overview.md new file mode 100644 index 0000000..b5d9ac3 --- /dev/null +++ b/docs/cli/overview.md @@ -0,0 +1,335 @@ +# CLI Overview + +This guide covers common features and configuration methods for the `airoa-lineage` CLI interfaces. + +## Available CLI Commands + +- **[airoa-lineage-usb-copy](usb-copy.md)** - Track USB data copy sessions +- **[airoa-lineage-wasabi-upload](wasabi-upload.md)** - Track Wasabi upload sessions + +--- + +## Installation + +```bash +# Using uv (recommended) +uv pip install -e . + +# Or using pip +pip install -e . +``` + +Verify installation: + +```bash +airoa-lineage-usb-copy --version +airoa-lineage-wasabi-upload --version +``` + +--- + +## Configuration + +The CLI supports three configuration sources with the following priority: + +**CLI arguments > Environment variables > Configuration file > Defaults** + +### Configuration File + +Default path: `~/.config/airoa-lineage/config.json` + +You can override this with the `--config` option or by setting `XDG_CONFIG_HOME`. + +**Example config.json:** + +```json +{ + "marquez_url": "http://localhost:9000", + "namespace": "airoa_examples", + "facet_prefix": "airoa", + "job_name": "usb-data-copy", + "common_facet": { + "robotId": "hsr001", + "location": "weblab", + "repositoryHash": "df110d5a8e3b9c2f1a7d6e4f5c3a2b1", + "repositoryUri": "https://github.com/AIRoA/airoa-lineage.git", + "repositoryTag": "v1.0.0", + "repositoryBranch": "main" + } +} +``` + +**Configuration Schema:** + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `marquez_url` | string | No | `http://localhost:9000` | Marquez server URL | +| `namespace` | string | **Yes** | - | OpenLineage namespace | +| `job_name` | string | No | Command-specific | Job identifier | +| `facet_prefix` | string | No | `""` (empty) | Prefix for custom facets | +| `common_facet` | object | **Yes** | - | CommonRunFacet fields | + +**CommonRunFacet Fields:** + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `robotId` | string | **Yes** | Robot identifier (e.g., `hsr001`) | +| `location` | string | **Yes** | Location identifier (e.g., `weblab`) | +| `repositoryHash` | string | **Yes** | Git commit hash | +| `repositoryUri` | string | **Yes** | Repository URI | +| `repositoryTag` | string | **Yes** | Git tag (e.g., `v1.0.0`) | +| `repositoryBranch` | string | **Yes** | Git branch (e.g., `main`) | + +### Environment Variables + +All configuration values can be set via environment variables: + +```bash +# Top-level configuration +export AIROA_MARQUEZ_URL=http://localhost:9000 +export AIROA_NAMESPACE=airoa_examples +export AIROA_FACET_PREFIX=airoa +export AIROA_JOB_NAME=usb-data-copy + +# CommonRunFacet fields +export AIROA_ROBOT_ID=hsr001 +export AIROA_LOCATION=weblab +export AIROA_REPOSITORY_HASH=$(git rev-parse HEAD) +export AIROA_REPOSITORY_URI=https://github.com/AIRoA/airoa-lineage.git +export AIROA_REPOSITORY_TAG=$(git describe --tags) +export AIROA_REPOSITORY_BRANCH=$(git rev-parse --abbrev-ref HEAD) +``` + +**Note:** `MARQUEZ_URL` is also supported for backward compatibility, but `AIROA_MARQUEZ_URL` takes precedence. + +### Priority Order Example + +```bash +# config.json has: "namespace": "dev" +# Environment has: AIROA_NAMESPACE=staging + +# This uses CLI argument (production) +airoa-lineage-usb-copy start --namespace production + +# This uses environment variable (staging) +airoa-lineage-usb-copy start + +# If no env var, uses config file (dev) +unset AIROA_NAMESPACE +airoa-lineage-usb-copy start +``` + +--- + +## Global Options + +These options apply to all commands: + +| Option | Short | Description | +|--------|-------|-------------| +| `--config PATH` | - | Path to config file (default: `~/.config/airoa-lineage/config.json`) | +| `--verbose` | `-v` | Enable verbose output (messages to stderr) | +| `--quiet` | `-q` | Suppress output except errors | +| `--json` | - | Output in JSON format | +| `--version` | - | Show version and exit | +| `--help` | `-h` | Show help message and exit | + +**Examples:** + +```bash +# Show version +airoa-lineage-usb-copy --version + +# Show help +airoa-lineage-usb-copy --help +airoa-lineage-usb-copy start --help + +# Use custom config file +airoa-lineage-usb-copy --config /path/to/config.json start + +# Verbose mode +airoa-lineage-usb-copy -v start + +# JSON output +airoa-lineage-usb-copy --json start +``` + +--- + +## Output Modes + +### Default Mode + +- **start:** Prints only `run_id` to stdout (for piping) +- **complete/cancel:** Prints success message to stdout + +```bash +RUN_ID=$(airoa-lineage-usb-copy start) +# Output: 550e8400-e29b-41d4-a716-446655440000 +``` + +### Verbose Mode (`-v`, `--verbose`) + +- All messages go to **stderr** +- `run_id` goes to **stdout** (for piping) +- Includes configuration details and event status + +```bash +airoa-lineage-usb-copy -v start +# stderr: Loading config from ~/.config/airoa-lineage/config.json +# stderr: Connecting to Marquez at http://localhost:9000 +# stderr: Session started successfully +# stdout: 550e8400-e29b-41d4-a716-446655440000 +``` + +### Quiet Mode (`-q`, `--quiet`) + +- **start:** Prints only `run_id` to stdout +- **complete/cancel:** No output (silent) +- Errors still go to stderr + +```bash +airoa-lineage-usb-copy -q complete --run-id "$RUN_ID" +# (no output) +``` + +### JSON Mode (`--json`) + +- All output in JSON format +- Suitable for programmatic parsing + +```bash +airoa-lineage-usb-copy --json start +# {"run_id": "550e8400-e29b-41d4-a716-446655440000", "status": "started"} + +airoa-lineage-usb-copy --json complete --run-id "$RUN_ID" +# {"run_id": "550e8400-e29b-41d4-a716-446655440000", "status": "completed"} +``` + +--- + +## Exit Codes + +The CLI uses standard exit codes to indicate success or failure: + +| Code | Meaning | Description | +|------|---------|-------------| +| `0` | Success | Command completed successfully | +| `1` | General error | Session error, runtime error, or unknown error | +| `2` | Connection error | Failed to connect to Marquez server | +| `3` | Configuration error | Missing or invalid configuration | + +**Example:** + +```bash +airoa-lineage-usb-copy start +EXIT_CODE=$? + +if [ $EXIT_CODE -eq 0 ]; then + echo "Success" +elif [ $EXIT_CODE -eq 3 ]; then + echo "Configuration error - check your config file" +elif [ $EXIT_CODE -eq 2 ]; then + echo "Connection error - is Marquez running?" +else + echo "Unknown error" +fi +``` + +--- + +## Troubleshooting + +### Common Errors + +#### 1. "Missing required CommonRunFacet fields" + +**Solution:** + +Provide all required fields via one of these methods: + +```bash +# Option 1: CLI arguments +airoa-lineage-usb-copy start \ + --robot-id hsr001 \ + --location weblab \ + --repository-hash $(git rev-parse HEAD) \ + --repository-uri https://github.com/AIRoA/airoa-lineage.git \ + --repository-tag v1.0.0 \ + --repository-branch main + +# Option 2: Environment variables +export AIROA_ROBOT_ID=hsr001 +export AIROA_LOCATION=weblab +# ... other fields + +# Option 3: Config file +# Add common_facet to ~/.config/airoa-lineage/config.json +``` + +#### 2. "namespace is required" + +**Solution:** + +```bash +# Option 1: CLI argument +airoa-lineage-usb-copy start --namespace airoa_examples + +# Option 2: Environment variable +export AIROA_NAMESPACE=airoa_examples + +# Option 3: Config file +echo '{"namespace": "airoa_examples"}' > ~/.config/airoa-lineage/config.json +``` + +#### 3. Connection refused / Marquez server not running + +**Solution:** + +Start the Marquez server: + +```bash +# Using Docker +docker run -d -p 3000:3000 -p 9000:9000 marquezproject/marquez:latest + +# Verify it's running +curl http://localhost:9000/api/v1/namespaces +``` + +### Debugging Tips + +#### Use verbose mode + +```bash +airoa-lineage-usb-copy -v start +``` + +This shows config file path, loaded configuration values, Marquez connection details, and event emission status. + +#### Use dry-run mode + +```bash +airoa-lineage-usb-copy start --dry-run +``` + +This shows what would be sent to Marquez without actually sending it. + +--- + +## Examples + +For complete working examples, see the respective documentation pages: + +- [USBCopy CLI Examples](usb-copy.md#examples) +- [WasabiUpload CLI Examples](wasabi-upload.md#examples) + +--- + +## Getting Help + +- CLI help: `airoa-lineage-usb-copy --help` or `airoa-lineage-wasabi-upload --help` +- Command help: ` start --help` +- Version: ` --version` +- GitHub Issues: https://github.com/AIRoA/airoa-lineage/issues +- OpenLineage docs: https://openlineage.io/docs +- Marquez docs: https://marquezproject.github.io/marquez/ diff --git a/docs/cli/usb-copy.md b/docs/cli/usb-copy.md new file mode 100644 index 0000000..7e52468 --- /dev/null +++ b/docs/cli/usb-copy.md @@ -0,0 +1,266 @@ +# USBCopy CLI + +The `airoa-lineage-usb-copy` command provides a CLI interface for tracking USB data copy operations using OpenLineage. + +## Commands + +### `start` - Start USB Copy Session + +Start a USB copy session and emit a START event to Marquez. + +**Synopsis:** + +```bash +airoa-lineage-usb-copy start [OPTIONS] +``` + +**Options:** + +| Option | Type | Required | Description | +|--------|------|----------|-------------| +| `--namespace` | string | Yes* | OpenLineage namespace | +| `--robot-id` | string | Yes* | Robot identifier | +| `--location` | string | Yes* | Location identifier | +| `--repository-hash` | string | Yes* | Git commit hash | +| `--repository-uri` | string | Yes* | Repository URI | +| `--repository-tag` | string | Yes* | Git tag | +| `--repository-branch` | string | Yes* | Git branch | +| `--nominal-start-time` | string | No | Nominal start time (ISO 8601 format) | +| `--nominal-end-time` | string | No | Nominal end time (ISO 8601 format) | +| `--job-name` | string | No | Job name (default: `usb-data-copy`) | +| `--marquez-url` | string | No | Marquez server URL | +| `--facet-prefix` | string | No | Facet prefix | +| `--dry-run` | flag | No | Show what would be done without executing | + +\* Required unless provided via config file or environment variables + +**Output:** + +- **Default:** Prints the `run_id` (UUID) to stdout +- **JSON mode:** `{"run_id": "...", "status": "started"}` +- **Verbose mode:** Additional messages to stderr, run_id to stdout +- **Dry-run mode:** Shows configuration without emitting events + +**Examples:** + +```bash +# Basic usage +RUN_ID=$(airoa-lineage-usb-copy start \ + --namespace airoa_examples \ + --robot-id hsr001 \ + --location weblab \ + --repository-hash $(git rev-parse HEAD) \ + --repository-uri https://github.com/AIRoA/airoa-lineage.git \ + --repository-tag v1.0.0 \ + --repository-branch main) + +# With nominal times (for batch processing) +RUN_ID=$(airoa-lineage-usb-copy start \ + --nominal-start-time "2025-11-26T00:00:00+00:00" \ + --nominal-end-time "2025-11-26T05:00:00+00:00") + +# Dry-run mode (verify configuration) +airoa-lineage-usb-copy start --dry-run + +# JSON output +airoa-lineage-usb-copy start --json +``` + +--- + +### `complete` - Complete USB Copy Session + +Complete a USB copy session and emit a COMPLETE event to Marquez. + +**Synopsis:** + +```bash +airoa-lineage-usb-copy complete --run-id RUN_ID [OPTIONS] +``` + +**Options:** + +| Option | Type | Required | Description | +|--------|------|----------|-------------| +| `--run-id` | string | **Yes** | Session run ID (from start command) | +| `--namespace` | string | Yes* | OpenLineage namespace | +| `--job-name` | string | No | Job name | +| `--marquez-url` | string | No | Marquez server URL | +| `--facet-prefix` | string | No | Facet prefix | +| `--robot-id` | string | Yes* | Robot identifier | +| `--location` | string | Yes* | Location identifier | +| `--repository-hash` | string | Yes* | Git commit hash | +| `--repository-uri` | string | Yes* | Repository URI | +| `--repository-tag` | string | Yes* | Git tag | +| `--repository-branch` | string | Yes* | Git branch | + +\* Required unless provided via config file or environment variables + +**Important:** It is recommended to provide the same `namespace`, `job_name`, and CommonRunFacet values as used in the `start` command. If different values are provided, OpenLineage will treat them as separate jobs. + +**Output:** + +- **Default:** `Session completed successfully` +- **JSON mode:** `{"run_id": "...", "status": "completed"}` +- **Quiet mode:** No output + +**Examples:** + +```bash +# Basic usage +airoa-lineage-usb-copy complete --run-id "$RUN_ID" + +# Specify all arguments explicitly (using same values as start) +airoa-lineage-usb-copy complete \ + --run-id "$RUN_ID" \ + --namespace airoa_examples \ + --job-name usb-data-copy \ + --robot-id hsr001 \ + --location weblab \ + --repository-hash abc123 \ + --repository-uri https://github.com/AIRoA/airoa-lineage.git \ + --repository-tag v1.0.0 \ + --repository-branch main + +# JSON output +airoa-lineage-usb-copy complete --run-id "$RUN_ID" --json + +# Quiet mode +airoa-lineage-usb-copy complete --run-id "$RUN_ID" --quiet +``` + +--- + +### `cancel` - Cancel USB Copy Session + +Cancel a USB copy session and emit an ABORT event to Marquez. + +**Synopsis:** + +```bash +airoa-lineage-usb-copy cancel --run-id RUN_ID [OPTIONS] +``` + +**Options:** + +| Option | Type | Required | Description | +|--------|------|----------|-------------| +| `--run-id` | string | **Yes** | Session run ID (from start command) | +| `--namespace` | string | Yes* | OpenLineage namespace | +| `--job-name` | string | No | Job name | +| `--marquez-url` | string | No | Marquez server URL | +| `--facet-prefix` | string | No | Facet prefix | +| `--robot-id` | string | Yes* | Robot identifier | +| `--location` | string | Yes* | Location identifier | +| `--repository-hash` | string | Yes* | Git commit hash | +| `--repository-uri` | string | Yes* | Repository URI | +| `--repository-tag` | string | Yes* | Git tag | +| `--repository-branch` | string | Yes* | Git branch | + +\* Required unless provided via config file or environment variables + +**Important:** It is recommended to provide the same `namespace`, `job_name`, and CommonRunFacet values as used in the `start` command. If different values are provided, OpenLineage will treat them as separate jobs. + +**Output:** + +- **Default:** `Session cancelled successfully` +- **JSON mode:** `{"run_id": "...", "status": "cancelled"}` +- **Quiet mode:** No output + +**Examples:** + +```bash +# Basic usage +airoa-lineage-usb-copy cancel --run-id "$RUN_ID" + +# Specify all arguments explicitly (using same values as start) +airoa-lineage-usb-copy cancel \ + --run-id "$RUN_ID" \ + --namespace airoa_examples \ + --job-name usb-data-copy \ + --robot-id hsr001 \ + --location weblab \ + --repository-hash abc123 \ + --repository-uri https://github.com/AIRoA/airoa-lineage.git \ + --repository-tag v1.0.0 \ + --repository-branch main + +# JSON output +airoa-lineage-usb-copy cancel --run-id "$RUN_ID" --json + +# Quiet mode +airoa-lineage-usb-copy cancel --run-id "$RUN_ID" --quiet +``` + +--- + +## Examples + +For complete working examples, see [examples/cli/usb_copy/](../../examples/cli/usb_copy/). + +### Available Examples + +The following executable shell scripts demonstrate different usage patterns: + +1. **[basic_workflow.sh](../../examples/cli/usb_copy/basic_workflow.sh)** - Simple start → complete workflow + - All configuration via CLI arguments + - Ideal for quick testing and learning + +2. **[with_config_file.sh](../../examples/cli/usb_copy/with_config_file.sh)** - Configuration file usage + - Loads settings from `config.example.json` + - Minimal CLI arguments required + - Includes dry-run mode for testing + +3. **[with_env_vars.sh](../../examples/cli/usb_copy/with_env_vars.sh)** - Environment variables + - Configuration via `AIROA_*` environment variables + - Dynamically retrieves repository info from git + +4. **[error_handling.sh](../../examples/cli/usb_copy/error_handling.sh)** - Error handling with trap + - Automatic session cancellation on error + - Cleanup function with proper exit codes + +5. **[retry_logic.sh](../../examples/cli/usb_copy/retry_logic.sh)** - Connection retry logic + - Automatic retry on connection errors + - Maximum 3 attempts with exponential backoff + +See [examples/cli/usb_copy/README.md](../../examples/cli/usb_copy/README.md) for detailed documentation on each example, including prerequisites, usage instructions, and integration patterns. + +--- + +## Quick Start + +```bash +# 1. Start a USB copy session +RUN_ID=$(airoa-lineage-usb-copy start \ + --namespace airoa_examples \ + --robot-id hsr001 \ + --location weblab \ + --repository-hash $(git rev-parse HEAD) \ + --repository-uri https://github.com/AIRoA/airoa-lineage.git \ + --repository-tag v1.0.0 \ + --repository-branch main \ + --nominal-start-time "2025-11-26T00:00:00+00:00" \ + --nominal-end-time "2025-11-26T05:00:00+00:00") + +# 2. Perform your USB copy operation +# (your data copy processing here) + +# 3. Complete the session (using same metadata as start) +airoa-lineage-usb-copy complete \ + --run-id "$RUN_ID" \ + --namespace airoa_examples \ + --job-name usb-data-copy \ + --robot-id hsr001 \ + --location weblab \ + --repository-hash $(git rev-parse HEAD) \ + --repository-uri https://github.com/AIRoA/airoa-lineage.git \ + --repository-tag v1.0.0 \ + --repository-branch main +``` + +--- + +## Related Documentation + +- [CLI Overview](overview.md) - Common configuration, environment variables, troubleshooting +- [WasabiUpload CLI](wasabi-upload.md) - Track Wasabi upload operations diff --git a/docs/cli/wasabi-upload.md b/docs/cli/wasabi-upload.md new file mode 100644 index 0000000..cc4eb19 --- /dev/null +++ b/docs/cli/wasabi-upload.md @@ -0,0 +1,282 @@ +# WasabiUpload CLI + +The `airoa-lineage-wasabi-upload` command provides a CLI interface for tracking Wasabi data upload operations using OpenLineage. + +## Commands + +### `start` - Start Wasabi Upload Session + +Start a Wasabi upload session and emit a START event to Marquez. + +**Synopsis:** + +```bash +airoa-lineage-wasabi-upload start [OPTIONS] +``` + +**Options:** + +| Option | Type | Required | Description | +|--------|------|----------|-------------| +| `--namespace` | string | Yes* | OpenLineage namespace | +| `--robot-id` | string | Yes* | Robot identifier | +| `--location` | string | Yes* | Location identifier | +| `--repository-hash` | string | Yes* | Git commit hash | +| `--repository-uri` | string | Yes* | Repository URI | +| `--repository-tag` | string | Yes* | Git tag | +| `--repository-branch` | string | Yes* | Git branch | +| `--nominal-start-time` | string | No | Nominal start time (ISO 8601 format) | +| `--nominal-end-time` | string | No | Nominal end time (ISO 8601 format) | +| `--job-name` | string | No | Job name (default: `wasabi-data-upload`) | +| `--marquez-url` | string | No | Marquez server URL | +| `--facet-prefix` | string | No | Facet prefix | +| `--dry-run` | flag | No | Show what would be done without executing | + +\* Required unless provided via config file or environment variables + +**Output:** + +- **Default:** Prints the `run_id` (UUID) to stdout +- **JSON mode:** `{"run_id": "...", "status": "started"}` +- **Verbose mode:** Additional messages to stderr, run_id to stdout +- **Dry-run mode:** Shows configuration without emitting events + +**Examples:** + +```bash +# Basic usage +RUN_ID=$(airoa-lineage-wasabi-upload start \ + --namespace airoa_examples \ + --robot-id hsr001 \ + --location weblab \ + --repository-hash $(git rev-parse HEAD) \ + --repository-uri https://github.com/AIRoA/airoa-lineage.git \ + --repository-tag v1.0.0 \ + --repository-branch main) + +# With nominal times (for batch processing) +RUN_ID=$(airoa-lineage-wasabi-upload start \ + --nominal-start-time "2025-11-26T00:00:00+00:00" \ + --nominal-end-time "2025-11-26T05:00:00+00:00") + +# With custom job name +RUN_ID=$(airoa-lineage-wasabi-upload start \ + --job-name "wasabi-experiment-upload") + +# Dry-run mode (verify configuration) +airoa-lineage-wasabi-upload start --dry-run + +# JSON output +RUN_ID=$(airoa-lineage-wasabi-upload start --json | jq -r '.run_id') + +# Verbose mode +airoa-lineage-wasabi-upload start --verbose +``` + +--- + +### `complete` - Complete Wasabi Upload Session + +Complete a Wasabi upload session and emit a COMPLETE event to Marquez. + +**Synopsis:** + +```bash +airoa-lineage-wasabi-upload complete --run-id RUN_ID [OPTIONS] +``` + +**Options:** + +| Option | Type | Required | Description | +|--------|------|----------|-------------| +| `--run-id` | string | **Yes** | Session run ID (from start command) | +| `--namespace` | string | Yes* | OpenLineage namespace | +| `--job-name` | string | No | Job name | +| `--marquez-url` | string | No | Marquez server URL | +| `--facet-prefix` | string | No | Facet prefix | +| `--robot-id` | string | Yes* | Robot identifier | +| `--location` | string | Yes* | Location identifier | +| `--repository-hash` | string | Yes* | Git commit hash | +| `--repository-uri` | string | Yes* | Repository URI | +| `--repository-tag` | string | Yes* | Git tag | +| `--repository-branch` | string | Yes* | Git branch | + +\* Required unless provided via config file or environment variables + +**Important:** It is recommended to provide the same `namespace`, `job_name`, and CommonRunFacet values as used in the `start` command. If different values are provided, OpenLineage will treat them as separate jobs. + +**Output:** + +- **Default:** `Session completed successfully` +- **JSON mode:** `{"run_id": "...", "status": "completed"}` +- **Quiet mode:** No output + +**Examples:** + +```bash +# Basic usage +airoa-lineage-wasabi-upload complete --run-id "$RUN_ID" + +# With namespace override +airoa-lineage-wasabi-upload complete --run-id "$RUN_ID" --namespace airoa_examples + +# Specify all arguments explicitly (using same values as start) +airoa-lineage-wasabi-upload complete \ + --run-id "$RUN_ID" \ + --namespace airoa_examples \ + --job-name wasabi-data-upload \ + --robot-id hsr001 \ + --location weblab \ + --repository-hash abc123 \ + --repository-uri https://github.com/AIRoA/airoa-lineage.git \ + --repository-tag v1.0.0 \ + --repository-branch main + +# JSON output +airoa-lineage-wasabi-upload complete --run-id "$RUN_ID" --json + +# Quiet mode +airoa-lineage-wasabi-upload complete --run-id "$RUN_ID" --quiet +``` + +--- + +### `cancel` - Cancel Wasabi Upload Session + +Cancel a Wasabi upload session and emit an ABORT event to Marquez. + +**Synopsis:** + +```bash +airoa-lineage-wasabi-upload cancel --run-id RUN_ID [OPTIONS] +``` + +**Options:** + +| Option | Type | Required | Description | +|--------|------|----------|-------------| +| `--run-id` | string | **Yes** | Session run ID (from start command) | +| `--namespace` | string | Yes* | OpenLineage namespace | +| `--job-name` | string | No | Job name | +| `--marquez-url` | string | No | Marquez server URL | +| `--facet-prefix` | string | No | Facet prefix | +| `--robot-id` | string | Yes* | Robot identifier | +| `--location` | string | Yes* | Location identifier | +| `--repository-hash` | string | Yes* | Git commit hash | +| `--repository-uri` | string | Yes* | Repository URI | +| `--repository-tag` | string | Yes* | Git tag | +| `--repository-branch` | string | Yes* | Git branch | + +\* Required unless provided via config file or environment variables + +**Important:** It is recommended to provide the same `namespace`, `job_name`, and CommonRunFacet values as used in the `start` command. If different values are provided, OpenLineage will treat them as separate jobs. + +**Output:** + +- **Default:** `Session cancelled successfully` +- **JSON mode:** `{"run_id": "...", "status": "cancelled"}` +- **Quiet mode:** No output + +**Examples:** + +```bash +# Basic usage +airoa-lineage-wasabi-upload cancel --run-id "$RUN_ID" + +# Override specific CommonRunFacet fields +airoa-lineage-wasabi-upload cancel \ + --run-id "$RUN_ID" \ + --robot-id hsr002 \ + --location lab2 + +# Specify all arguments explicitly (using same values as start) +airoa-lineage-wasabi-upload cancel \ + --run-id "$RUN_ID" \ + --namespace airoa_examples \ + --job-name wasabi-data-upload \ + --robot-id hsr001 \ + --location weblab \ + --repository-hash abc123 \ + --repository-uri https://github.com/AIRoA/airoa-lineage.git \ + --repository-tag v1.0.0 \ + --repository-branch main + +# JSON output +airoa-lineage-wasabi-upload cancel --run-id "$RUN_ID" --json + +# Quiet mode +airoa-lineage-wasabi-upload cancel --run-id "$RUN_ID" --quiet +``` + +--- + +## Examples + +For complete working examples, see [examples/cli/wasabi_upload/](../../examples/cli/wasabi_upload/). + +### Available Examples + +The following executable shell scripts demonstrate different usage patterns: + +1. **[basic_workflow.sh](../../examples/cli/wasabi_upload/basic_workflow.sh)** - Simple start → complete workflow + - All configuration via CLI arguments + - Ideal for quick testing and learning + +2. **[with_config_file.sh](../../examples/cli/wasabi_upload/with_config_file.sh)** - Configuration file usage + - Loads settings from `config.example.json` + - Minimal CLI arguments required + - Includes dry-run mode for testing + +3. **[with_env_vars.sh](../../examples/cli/wasabi_upload/with_env_vars.sh)** - Environment variables + - Configuration via `AIROA_*` environment variables + - Dynamically retrieves repository info from git + +4. **[error_handling.sh](../../examples/cli/wasabi_upload/error_handling.sh)** - Error handling with trap + - Automatic session cancellation on error + - Cleanup function with proper exit codes + +5. **[retry_logic.sh](../../examples/cli/wasabi_upload/retry_logic.sh)** - Connection retry logic + - Automatic retry on connection errors + - Maximum 3 attempts with exponential backoff + +See [examples/cli/wasabi_upload/README.md](../../examples/cli/wasabi_upload/README.md) for detailed documentation on each example, including prerequisites, usage instructions, and integration patterns. + +--- + +## Quick Start + +```bash +# 1. Start a Wasabi upload session +RUN_ID=$(airoa-lineage-wasabi-upload start \ + --namespace airoa_examples \ + --robot-id hsr001 \ + --location weblab \ + --repository-hash $(git rev-parse HEAD) \ + --repository-uri https://github.com/AIRoA/airoa-lineage.git \ + --repository-tag v1.0.0 \ + --repository-branch main \ + --nominal-start-time "2025-11-26T00:00:00+00:00" \ + --nominal-end-time "2025-11-26T05:00:00+00:00") + +# 2. Perform your Wasabi upload operation +# (your upload processing here) + +# 3. Complete the session (using same metadata as start) +airoa-lineage-wasabi-upload complete \ + --run-id "$RUN_ID" \ + --namespace airoa_examples \ + --job-name wasabi-data-upload \ + --robot-id hsr001 \ + --location weblab \ + --repository-hash $(git rev-parse HEAD) \ + --repository-uri https://github.com/AIRoA/airoa-lineage.git \ + --repository-tag v1.0.0 \ + --repository-branch main +``` + +--- + +## Related Documentation + +- [CLI Overview](overview.md) - Common configuration, environment variables, troubleshooting +- [USBCopy CLI](usb-copy.md) - Track USB copy operations diff --git a/examples/cli/usb_copy/basic_workflow.sh b/examples/cli/usb_copy/basic_workflow.sh index 1904729..723f4ad 100755 --- a/examples/cli/usb_copy/basic_workflow.sh +++ b/examples/cli/usb_copy/basic_workflow.sh @@ -17,18 +17,32 @@ set -e # Exit immediately if a command exits with a non-zero status +# Ensure uv is in PATH +export PATH="/opt/homebrew/bin:$PATH" + +# Common arguments shared between start and complete commands +NAMESPACE="airoa_examples" +JOB_NAME="usb-data-copy" +ROBOT_ID="hsr001" +LOCATION="weblab" +REPOSITORY_HASH=$(git rev-parse HEAD) +REPOSITORY_URI="https://github.com/AIRoA/airoa-lineage.git" +REPOSITORY_TAG="v1.0.0" +REPOSITORY_BRANCH="main" + # 1. Start a USB copy session # The start command returns a RUN_ID which uniquely identifies this session echo "Starting USB copy session..." -RUN_ID=$(airoa-lineage-usb-copy start \ - --namespace production \ - --robot-id hsr001 \ - --location weblab \ - --repository-hash $(git rev-parse HEAD) \ - --repository-uri https://github.com/AIRoA/airoa-lineage.git \ - --repository-tag v1.0.0 \ - --repository-branch main \ +RUN_ID=$(uv run airoa-lineage-usb-copy start \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH" \ --nominal-start-time "2025-11-26T00:00:00+00:00" \ --nominal-end-time "2025-11-26T05:00:00+00:00") @@ -41,7 +55,17 @@ echo "Performing USB copy operation... (this is a placeholder)" # 3. Complete the session # Mark the session as successfully completed +# IMPORTANT: Use the same namespace, job_name, and common_facet as the start command echo "Completing USB copy session..." -airoa-lineage-usb-copy complete --run-id "$RUN_ID" +uv run airoa-lineage-usb-copy complete \ + --run-id "$RUN_ID" \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH" echo "USB copy session completed successfully!" diff --git a/examples/cli/usb_copy/config.example.json b/examples/cli/usb_copy/config.example.json index ad01f10..986732f 100644 --- a/examples/cli/usb_copy/config.example.json +++ b/examples/cli/usb_copy/config.example.json @@ -1,5 +1,5 @@ { - "namespace": "airoa_production", + "namespace": "airoa_examples", "marquez_url": "http://localhost:9000", "job_name": "usb-data-copy", "facet_prefix": "airoa", diff --git a/examples/cli/usb_copy/error_handling.sh b/examples/cli/usb_copy/error_handling.sh index 1e049fb..7cc5cd9 100755 --- a/examples/cli/usb_copy/error_handling.sh +++ b/examples/cli/usb_copy/error_handling.sh @@ -20,9 +20,22 @@ # - Proper error code handling ################################################################################ +# Ensure uv is in PATH +export PATH="/opt/homebrew/bin:$PATH" + # Don't use 'set -e' here - we want to handle errors ourselves set -u # Exit on undefined variable +# Common arguments shared between start, complete, and cancel commands +NAMESPACE="airoa_examples" +JOB_NAME="usb-data-copy" +ROBOT_ID="hsr001" +LOCATION="weblab" +REPOSITORY_HASH=$(git rev-parse HEAD) +REPOSITORY_URI="https://github.com/AIRoA/airoa-lineage.git" +REPOSITORY_TAG="v1.0.0" +REPOSITORY_BRANCH="main" + # Global variable to store RUN_ID RUN_ID="" @@ -37,7 +50,17 @@ cleanup() { echo "Cancelling USB copy session..." # Cancel the session on error - if airoa-lineage-usb-copy cancel --run-id "$RUN_ID"; then + # IMPORTANT: Use the same namespace, job_name, and common_facet as the start command + if uv run airoa-lineage-usb-copy cancel \ + --run-id "$RUN_ID" \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH"; then echo "Session cancelled successfully" else echo "Warning: Failed to cancel session" @@ -51,14 +74,15 @@ trap cleanup EXIT # Start session echo "Starting USB copy session..." -RUN_ID=$(airoa-lineage-usb-copy start \ - --namespace production \ - --robot-id hsr001 \ - --location weblab \ - --repository-hash $(git rev-parse HEAD) \ - --repository-uri https://github.com/AIRoA/airoa-lineage.git \ - --repository-tag v1.0.0 \ - --repository-branch main \ +RUN_ID=$(uv run airoa-lineage-usb-copy start \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH" \ --nominal-start-time "2025-11-26T00:00:00+00:00" \ --nominal-end-time "2025-11-26T05:00:00+00:00") @@ -79,8 +103,18 @@ echo "Performing USB copy operation..." echo "USB copy operation completed" # Complete the session +# IMPORTANT: Use the same namespace, job_name, and common_facet as the start command echo "Completing session..." -if ! airoa-lineage-usb-copy complete --run-id "$RUN_ID"; then +if ! uv run airoa-lineage-usb-copy complete \ + --run-id "$RUN_ID" \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH"; then echo "Error: Failed to complete session" exit 1 fi diff --git a/examples/cli/usb_copy/retry_logic.sh b/examples/cli/usb_copy/retry_logic.sh index c5410df..863b061 100755 --- a/examples/cli/usb_copy/retry_logic.sh +++ b/examples/cli/usb_copy/retry_logic.sh @@ -20,8 +20,21 @@ # - Exponential backoff between retries ################################################################################ +# Ensure uv is in PATH +export PATH="/opt/homebrew/bin:$PATH" + set -e +# Common arguments shared between start and complete commands +NAMESPACE="airoa_examples" +JOB_NAME="usb-data-copy" +ROBOT_ID="hsr001" +LOCATION="weblab" +REPOSITORY_HASH=$(git rev-parse HEAD) +REPOSITORY_URI="https://github.com/AIRoA/airoa-lineage.git" +REPOSITORY_TAG="v1.0.0" +REPOSITORY_BRANCH="main" + MAX_RETRIES=3 RETRY_DELAY=2 # seconds @@ -34,14 +47,15 @@ start_session_with_retry() { # Disable 'set -e' temporarily to capture exit code set +e - RUN_ID=$(airoa-lineage-usb-copy start \ - --namespace production \ - --robot-id hsr001 \ - --location weblab \ - --repository-hash $(git rev-parse HEAD) \ - --repository-uri https://github.com/AIRoA/airoa-lineage.git \ - --repository-tag v1.0.0 \ - --repository-branch main \ + RUN_ID=$(uv run airoa-lineage-usb-copy start \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH" \ --nominal-start-time "2025-11-26T00:00:00+00:00" \ --nominal-end-time "2025-11-26T05:00:00+00:00" 2>&1) local exit_code=$? @@ -86,13 +100,23 @@ echo "Performing USB copy operation..." # Your USB copy logic here # Complete the session with retry logic +# IMPORTANT: Use the same namespace, job_name, and common_facet as the start command echo "Completing session..." attempt=1 while [ $attempt -le $MAX_RETRIES ]; do echo "Attempt $attempt/$MAX_RETRIES: Completing session..." set +e - airoa-lineage-usb-copy complete --run-id "$RUN_ID" + uv run airoa-lineage-usb-copy complete \ + --run-id "$RUN_ID" \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH" exit_code=$? set -e diff --git a/examples/cli/usb_copy/with_config_file.sh b/examples/cli/usb_copy/with_config_file.sh index a6e4a6b..68dae33 100755 --- a/examples/cli/usb_copy/with_config_file.sh +++ b/examples/cli/usb_copy/with_config_file.sh @@ -34,11 +34,14 @@ # EOF ################################################################################ +# Ensure uv is in PATH +export PATH="/opt/homebrew/bin:$PATH" + set -e # Test configuration with dry-run mode first echo "Testing configuration with dry-run mode..." -airoa-lineage-usb-copy start \ +uv run airoa-lineage-usb-copy start \ --nominal-start-time "2025-11-26T00:00:00+00:00" \ --nominal-end-time "2025-11-26T05:00:00+00:00" \ --dry-run @@ -48,7 +51,7 @@ echo "Configuration looks good! Starting actual session..." # Start session with minimal CLI arguments # All other settings are loaded from config file -RUN_ID=$(airoa-lineage-usb-copy start \ +RUN_ID=$(uv run airoa-lineage-usb-copy start \ --nominal-start-time "2025-11-26T00:00:00+00:00" \ --nominal-end-time "2025-11-26T05:00:00+00:00") @@ -60,6 +63,6 @@ echo "Performing USB copy operation..." # Complete the session echo "Completing session..." -airoa-lineage-usb-copy complete --run-id "$RUN_ID" +uv run airoa-lineage-usb-copy complete --run-id "$RUN_ID" echo "Session completed successfully!" diff --git a/examples/cli/usb_copy/with_env_vars.sh b/examples/cli/usb_copy/with_env_vars.sh index 1d0d41d..86338d8 100755 --- a/examples/cli/usb_copy/with_env_vars.sh +++ b/examples/cli/usb_copy/with_env_vars.sh @@ -16,10 +16,13 @@ # - Marquez server running (optional for dry-run mode) ################################################################################ +# Ensure uv is in PATH +export PATH="/opt/homebrew/bin:$PATH" + set -e # Export configuration via environment variables -export AIROA_NAMESPACE="production" +export AIROA_NAMESPACE="airoa_examples" export AIROA_MARQUEZ_URL="http://localhost:9000" export AIROA_JOB_NAME="usb-data-copy" export AIROA_FACET_PREFIX="airoa" @@ -52,7 +55,7 @@ echo " Repository Branch: $AIROA_REPOSITORY_BRANCH" echo "" # Start session - configuration is read from environment variables -RUN_ID=$(airoa-lineage-usb-copy start \ +RUN_ID=$(uv run airoa-lineage-usb-copy start \ --nominal-start-time "2025-11-26T00:00:00+00:00" \ --nominal-end-time "2025-11-26T05:00:00+00:00") @@ -63,7 +66,8 @@ echo "Performing USB copy operation..." # Your USB copy logic here # Complete the session +# IMPORTANT: Use the same namespace, job_name, and common_facet as the start command echo "Completing session..." -airoa-lineage-usb-copy complete --run-id "$RUN_ID" +uv run airoa-lineage-usb-copy complete --run-id "$RUN_ID" echo "Session completed successfully!" diff --git a/examples/cli/wasabi_upload/README.md b/examples/cli/wasabi_upload/README.md new file mode 100644 index 0000000..5aa4134 --- /dev/null +++ b/examples/cli/wasabi_upload/README.md @@ -0,0 +1,219 @@ +# Wasabi Upload CLI Examples + +Command-line interface examples for tracking Wasabi data upload operations with OpenLineage. + +## Prerequisites + +### Installation + +Install the airoa-lineage package: + +```bash +pip install airoa-lineage +``` + +Verify installation: + +```bash +airoa-lineage-wasabi-upload --version +``` + +### Marquez Server (Optional) + +A Marquez server is required for actual event tracking. However, you can test the CLI using `--dry-run` mode without a server. + +**To start a local Marquez server with Docker**: + +```bash +docker run -p 3000:3000 -p 9000:9000 marquezproject/marquez:latest +``` + +Access the Marquez UI at: http://localhost:3000 + +## Available Examples + +### 1. [basic_workflow.sh](basic_workflow.sh) + +**Purpose**: Demonstrates the simplest start → complete flow + +**Key Features**: +- All configuration via CLI arguments +- Clear step-by-step workflow +- Ideal for quick testing and learning + +**Usage**: +```bash +chmod +x basic_workflow.sh +./basic_workflow.sh +``` + +### 2. [with_config_file.sh](with_config_file.sh) + +**Purpose**: Use a configuration file to minimize CLI arguments + +**Key Features**: +- Loads settings from `~/.config/airoa-lineage/config.json` +- Minimal CLI arguments required +- Includes dry-run mode for testing configuration + +**Usage**: +```bash +# Set up configuration file first (see config.example.json) +mkdir -p ~/.config/airoa-lineage +cp config.example.json ~/.config/airoa-lineage/config.json +# Edit config.json with your settings + +chmod +x with_config_file.sh +./with_config_file.sh +``` + +### 3. [with_env_vars.sh](with_env_vars.sh) + +**Purpose**: Use environment variables for configuration + +**Key Features**: +- Configuration via `AIROA_*` environment variables +- Dynamically retrieves repository info from git commands +- Includes fallback handling for missing git tags + +**Usage**: +```bash +chmod +x with_env_vars.sh +./with_env_vars.sh +``` + +### 4. [error_handling.sh](error_handling.sh) + +**Purpose**: Demonstrate proper error handling with automatic cancellation + +**Key Features**: +- Uses `trap` for cleanup on error +- Automatically cancels sessions on failure +- Proper exit code handling + +**Usage**: +```bash +chmod +x error_handling.sh +./error_handling.sh +``` + +### 5. [retry_logic.sh](retry_logic.sh) + +**Purpose**: Handle connection errors with automatic retry + +**Key Features**: +- Automatic retry on connection errors (EXIT_CODE=2) +- Maximum 3 retry attempts with exponential backoff +- Useful for unreliable network conditions + +**Usage**: +```bash +chmod +x retry_logic.sh +./retry_logic.sh +``` + +## Configuration + +### Configuration File + +See [config.example.json](config.example.json) for a complete configuration file example. + +**Location**: `~/.config/airoa-lineage/config.json` + +**Required Fields**: +- `namespace` - OpenLineage namespace +- `marquez_url` - Marquez server URL +- `job_name` - Job name for tracking +- `common_facet` - Robot and repository metadata + - `robotId` - Robot identifier + - `location` - Robot location + - `repositoryHash` - Git commit hash + - `repositoryUri` - Git repository URL + - `repositoryTag` - Git tag + - `repositoryBranch` - Git branch + +### Environment Variables + +All configuration file fields can be set via environment variables with the `AIROA_` prefix: + +- `AIROA_NAMESPACE` +- `AIROA_MARQUEZ_URL` +- `AIROA_JOB_NAME` +- `AIROA_FACET_PREFIX` +- `AIROA_ROBOT_ID` +- `AIROA_LOCATION` +- `AIROA_REPOSITORY_HASH` +- `AIROA_REPOSITORY_URI` +- `AIROA_REPOSITORY_TAG` +- `AIROA_REPOSITORY_BRANCH` + +## Running Examples + +### Testing with Dry-Run Mode + +All examples can be tested without a Marquez server using `--dry-run`: + +```bash +airoa-lineage-wasabi-upload start \ + --namespace production \ + --robot-id hsr001 \ + --location weblab \ + --repository-hash $(git rev-parse HEAD) \ + --repository-uri https://github.com/AIRoA/airoa-lineage.git \ + --repository-tag v1.0.0 \ + --repository-branch main \ + --nominal-start-time "2025-11-26T00:00:00+00:00" \ + --nominal-end-time "2025-11-26T05:00:00+00:00" \ + --dry-run +``` + +This will display the configuration without sending events to Marquez. + +### Integration with Your Scripts + +To integrate Wasabi upload tracking into your existing scripts: + +1. **Start a session** before your Wasabi upload operation: + ```bash + RUN_ID=$(airoa-lineage-wasabi-upload start [options]) + ``` + +2. **Perform your Wasabi upload operation**: + ```bash + # Your actual Wasabi upload logic (rsync, cp, etc.) + rsync -av /source/path /destination/path + ``` + +3. **Complete or cancel** the session: + ```bash + # On success: + airoa-lineage-wasabi-upload complete --run-id "$RUN_ID" + + # On failure: + airoa-lineage-wasabi-upload cancel --run-id "$RUN_ID" + ``` + +## Troubleshooting + +### Common Issues + +**Error: Connection refused** +- Ensure Marquez server is running on the configured URL +- Check `marquez_url` in your configuration +- Try using `--dry-run` mode to test without server connection + +**Error: Invalid configuration** +- Verify all required fields are set in config file or environment variables +- Use `--verbose` flag to see detailed configuration loading +- Check configuration file syntax with a JSON validator + +**Error: Invalid RUN_ID format** +- Ensure RUN_ID is a valid UUID +- Verify the `start` command completed successfully +- Check that you're passing RUN_ID correctly to `complete`/`cancel` + +## Next Steps + +- Review the [CLI Usage Documentation](../../../docs/cli-usage.md) for complete API reference +- Learn about [OpenLineage concepts](https://openlineage.io/docs/) +- Explore the [Marquez UI](http://localhost:3000) to visualize your data lineage diff --git a/examples/cli/wasabi_upload/basic_workflow.sh b/examples/cli/wasabi_upload/basic_workflow.sh new file mode 100755 index 0000000..e69c6d3 --- /dev/null +++ b/examples/cli/wasabi_upload/basic_workflow.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +################################################################################ +# Basic Wasabi Upload Session Workflow +# +# Purpose: +# Demonstrates the simplest start → complete flow for Wasabi upload operations. +# All configuration is provided via CLI arguments. +# +# Usage: +# ./basic_workflow.sh +# +# Prerequisites: +# - airoa-lineage package installed (pip install airoa-lineage) +# - Marquez server running (optional for dry-run mode) +################################################################################ + +set -e # Exit immediately if a command exits with a non-zero status + +# Ensure uv is in PATH +export PATH="/opt/homebrew/bin:$PATH" + +# Common arguments shared between start and complete commands +NAMESPACE="airoa_examples" +JOB_NAME="wasabi-data-upload" +ROBOT_ID="hsr001" +LOCATION="weblab" +REPOSITORY_HASH=$(git rev-parse HEAD) +REPOSITORY_URI="https://github.com/AIRoA/airoa-lineage.git" +REPOSITORY_TAG="v1.0.0" +REPOSITORY_BRANCH="main" + +# 1. Start a Wasabi upload session +# The start command returns a RUN_ID which uniquely identifies this session +echo "Starting Wasabi upload session..." + +RUN_ID=$(uv run airoa-lineage-wasabi-upload start \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH" \ + --nominal-start-time "2025-11-26T00:00:00+00:00" \ + --nominal-end-time "2025-11-26T05:00:00+00:00") + +echo "Started Wasabi upload session with RUN_ID: $RUN_ID" + +# 2. Perform your Wasabi upload operation +# Replace this placeholder with your actual Wasabi upload logic +echo "Performing Wasabi upload operation... (this is a placeholder)" +# Example: rsync -av /source/path /destination/path + +# 3. Complete the session +# Mark the session as successfully completed +# IMPORTANT: Use the same namespace, job_name, and common_facet as the start command +echo "Completing Wasabi upload session..." +uv run airoa-lineage-wasabi-upload complete \ + --run-id "$RUN_ID" \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH" + +echo "Wasabi upload session completed successfully!" diff --git a/examples/cli/wasabi_upload/config.example.json b/examples/cli/wasabi_upload/config.example.json new file mode 100644 index 0000000..17adb6b --- /dev/null +++ b/examples/cli/wasabi_upload/config.example.json @@ -0,0 +1,14 @@ +{ + "namespace": "airoa_examples", + "marquez_url": "http://localhost:9000", + "job_name": "wasabi-data-upload", + "facet_prefix": "airoa", + "common_facet": { + "robotId": "hsr001", + "location": "weblab", + "repositoryHash": "df110d5b8c5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a", + "repositoryUri": "https://github.com/AIRoA/airoa-lineage.git", + "repositoryTag": "v1.0.0", + "repositoryBranch": "main" + } +} diff --git a/examples/cli/wasabi_upload/error_handling.sh b/examples/cli/wasabi_upload/error_handling.sh new file mode 100755 index 0000000..2e412a1 --- /dev/null +++ b/examples/cli/wasabi_upload/error_handling.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +################################################################################ +# Wasabi Upload Session with Error Handling +# +# Purpose: +# Demonstrates proper error handling using trap to automatically cancel +# sessions when errors occur. +# +# Usage: +# ./error_handling.sh +# +# Prerequisites: +# - airoa-lineage package installed +# - Configuration file or environment variables set up +# +# Key Features: +# - Automatic session cancellation on error + +# Ensure uv is in PATH +export PATH="/opt/homebrew/bin:$PATH" +# - Cleanup function with trap EXIT +# - Proper error code handling +################################################################################ + +# Don't use 'set -e' here - we want to handle errors ourselves +set -u # Exit on undefined variable + +# Common arguments shared between start, complete, and cancel commands +NAMESPACE="airoa_examples" +JOB_NAME="wasabi-data-upload" +ROBOT_ID="hsr001" +LOCATION="weblab" +REPOSITORY_HASH=$(git rev-parse HEAD) +REPOSITORY_URI="https://github.com/AIRoA/airoa-lineage.git" +REPOSITORY_TAG="v1.0.0" +REPOSITORY_BRANCH="main" + +# Global variable to store RUN_ID +RUN_ID="" + +# Cleanup function - called on script exit (success or failure) +cleanup() { + local EXIT_CODE=$? + + if [ -n "$RUN_ID" ]; then + if [ $EXIT_CODE -ne 0 ]; then + echo "" + echo "Error detected (exit code: $EXIT_CODE)" + echo "Cancelling Wasabi upload session..." + + # Cancel the session on error + # IMPORTANT: Use the same namespace, job_name, and common_facet as the start command + if uv run airoa-lineage-wasabi-upload cancel \ + --run-id "$RUN_ID" \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH"; then + echo "Session cancelled successfully" + else + echo "Warning: Failed to cancel session" + fi + fi + fi +} + +# Register cleanup function to run on exit +trap cleanup EXIT + +# Start session +echo "Starting Wasabi upload session..." +RUN_ID=$(uv run airoa-lineage-wasabi-upload start \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH" \ + --nominal-start-time "2025-11-26T00:00:00+00:00" \ + --nominal-end-time "2025-11-26T05:00:00+00:00") + +if [ -z "$RUN_ID" ]; then + echo "Error: Failed to start session (no RUN_ID returned)" + exit 1 +fi + +echo "Started Wasabi upload session with RUN_ID: $RUN_ID" + +# Simulate Wasabi upload operation (replace with actual logic) +echo "Performing Wasabi upload operation..." + +# Example: Uncomment the following line to simulate an error +# exit 1 + +# If we get here, operation was successful +echo "Wasabi upload operation completed" + +# Complete the session +# IMPORTANT: Use the same namespace, job_name, and common_facet as the start command +echo "Completing session..." +if ! uv run airoa-lineage-wasabi-upload complete \ + --run-id "$RUN_ID" \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH"; then + echo "Error: Failed to complete session" + exit 1 +fi + +# Clear RUN_ID to prevent cleanup from cancelling completed session +RUN_ID="" + +echo "Session completed successfully!" diff --git a/examples/cli/wasabi_upload/retry_logic.sh b/examples/cli/wasabi_upload/retry_logic.sh new file mode 100755 index 0000000..d09552e --- /dev/null +++ b/examples/cli/wasabi_upload/retry_logic.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +################################################################################ +# Wasabi Upload Session with Retry Logic +# +# Purpose: +# Demonstrates automatic retry logic for handling connection errors. +# Useful when Marquez server is temporarily unavailable. +# +# Usage: +# ./retry_logic.sh +# +# Prerequisites: +# - airoa-lineage package installed +# - Configuration file or environment variables set up +# +# Key Features: +# - Automatic retry on connection errors (EXIT_CODE=2) + +# Ensure uv is in PATH +export PATH="/opt/homebrew/bin:$PATH" +# - Maximum 3 retry attempts +# - Exponential backoff between retries +################################################################################ + +set -e + +# Common arguments shared between start and complete commands +NAMESPACE="airoa_examples" +JOB_NAME="wasabi-data-upload" +ROBOT_ID="hsr001" +LOCATION="weblab" +REPOSITORY_HASH=$(git rev-parse HEAD) +REPOSITORY_URI="https://github.com/AIRoA/airoa-lineage.git" +REPOSITORY_TAG="v1.0.0" +REPOSITORY_BRANCH="main" + +MAX_RETRIES=3 +RETRY_DELAY=2 # seconds + +# Function to start session with retry logic +start_session_with_retry() { + local attempt=1 + + while [ $attempt -le $MAX_RETRIES ]; do + echo "Attempt $attempt/$MAX_RETRIES: Starting Wasabi upload session..." + + # Disable 'set -e' temporarily to capture exit code + set +e + RUN_ID=$(uv run airoa-lineage-wasabi-upload start \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH" \ + --nominal-start-time "2025-11-26T00:00:00+00:00" \ + --nominal-end-time "2025-11-26T05:00:00+00:00" 2>&1) + local exit_code=$? + set -e + + # Check exit code + if [ $exit_code -eq 0 ]; then + echo "Successfully started session with RUN_ID: $RUN_ID" + return 0 + elif [ $exit_code -eq 2 ]; then + # EXIT_CODE=2 indicates connection error + echo "Connection error detected" + + if [ $attempt -lt $MAX_RETRIES ]; then + local wait_time=$((RETRY_DELAY * attempt)) + echo "Retrying in ${wait_time} seconds..." + sleep $wait_time + attempt=$((attempt + 1)) + else + echo "Error: Maximum retry attempts reached" + return 1 + fi + else + # Other errors - don't retry + echo "Error: Failed to start session (exit code: $exit_code)" + echo "$RUN_ID" + return 1 + fi + done + + return 1 +} + +# Start session with retry +if ! start_session_with_retry; then + echo "Failed to start Wasabi upload session after $MAX_RETRIES attempts" + exit 1 +fi + +# Perform Wasabi upload operation +echo "Performing Wasabi upload operation..." +# Your Wasabi upload logic here + +# Complete the session with retry logic +# IMPORTANT: Use the same namespace, job_name, and common_facet as the start command +echo "Completing session..." +attempt=1 +while [ $attempt -le $MAX_RETRIES ]; do + echo "Attempt $attempt/$MAX_RETRIES: Completing session..." + + set +e + uv run airoa-lineage-wasabi-upload complete \ + --run-id "$RUN_ID" \ + --namespace "$NAMESPACE" \ + --job-name "$JOB_NAME" \ + --robot-id "$ROBOT_ID" \ + --location "$LOCATION" \ + --repository-hash "$REPOSITORY_HASH" \ + --repository-uri "$REPOSITORY_URI" \ + --repository-tag "$REPOSITORY_TAG" \ + --repository-branch "$REPOSITORY_BRANCH" + exit_code=$? + set -e + + if [ $exit_code -eq 0 ]; then + echo "Session completed successfully!" + exit 0 + elif [ $exit_code -eq 2 ]; then + if [ $attempt -lt $MAX_RETRIES ]; then + wait_time=$((RETRY_DELAY * attempt)) + echo "Connection error. Retrying in ${wait_time} seconds..." + sleep $wait_time + attempt=$((attempt + 1)) + else + echo "Error: Maximum retry attempts reached" + exit 1 + fi + else + echo "Error: Failed to complete session (exit code: $exit_code)" + exit 1 + fi +done + +exit 1 diff --git a/examples/cli/wasabi_upload/with_config_file.sh b/examples/cli/wasabi_upload/with_config_file.sh new file mode 100755 index 0000000..4add68b --- /dev/null +++ b/examples/cli/wasabi_upload/with_config_file.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +################################################################################ +# Wasabi Upload Session with Configuration File +# +# Purpose: +# Demonstrates using a configuration file to minimize CLI arguments. +# Settings are loaded from ~/.config/airoa-lineage/config.json +# +# Usage: +# ./with_config_file.sh +# +# Prerequisites: +# - airoa-lineage package installed +# - Configuration file at ~/.config/airoa-lineage/config.json +# +# Configuration File Setup: +# mkdir -p ~/.config/airoa-lineage + +# Ensure uv is in PATH +export PATH="/opt/homebrew/bin:$PATH" +# cat > ~/.config/airoa-lineage/config.json </dev/null; then + export AIROA_REPOSITORY_TAG=$(git describe --exact-match --tags) +else + echo "Warning: No exact tag found for current commit, using 'untagged'" + export AIROA_REPOSITORY_TAG="untagged" +fi + +echo "Configuration from environment variables:" +echo " Namespace: $AIROA_NAMESPACE" +echo " Robot ID: $AIROA_ROBOT_ID" +echo " Location: $AIROA_LOCATION" +echo " Repository Hash: $AIROA_REPOSITORY_HASH" +echo " Repository URI: $AIROA_REPOSITORY_URI" +echo " Repository Tag: $AIROA_REPOSITORY_TAG" +echo " Repository Branch: $AIROA_REPOSITORY_BRANCH" +echo "" + +# Start session - configuration is read from environment variables +RUN_ID=$(uv run airoa-lineage-wasabi-upload start \ + --nominal-start-time "2025-11-26T00:00:00+00:00" \ + --nominal-end-time "2025-11-26T05:00:00+00:00") + +echo "Started Wasabi upload session with RUN_ID: $RUN_ID" + +# Perform Wasabi upload operation +echo "Performing Wasabi upload operation..." +# Your Wasabi upload logic here + +# Complete the session +echo "Completing session..." +uv run airoa-lineage-wasabi-upload complete --run-id "$RUN_ID" + +echo "Session completed successfully!" diff --git a/pyproject.toml b/pyproject.toml index 00a355a..b86fdf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ [project.scripts] airoa-lineage-usb-copy = "airoa_lineage.cli.usb_copy:main" +airoa-lineage-wasabi-upload = "airoa_lineage.cli.wasabi_upload:main" [dependency-groups] dev = [ diff --git a/src/airoa_lineage/cli/wasabi_upload.py b/src/airoa_lineage/cli/wasabi_upload.py new file mode 100644 index 0000000..3bf7b0a --- /dev/null +++ b/src/airoa_lineage/cli/wasabi_upload.py @@ -0,0 +1,501 @@ +"""CLI for WasabiUploadSession. + +This module provides a command-line interface for tracking Wasabi data upload +operations using OpenLineage. + +Usage: + airoa-lineage-wasabi-upload start [options] + airoa-lineage-wasabi-upload complete --run-id RUN_ID [options] + airoa-lineage-wasabi-upload cancel --run-id RUN_ID [options] +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Dict + +from airoa_lineage.cli.config import build_common_facet, load_config +from airoa_lineage.wasabi_upload import WasabiUploadSession + + +def _add_common_args(parser: argparse.ArgumentParser) -> None: + """ + Add common arguments shared across all commands. + + These arguments can be provided via CLI, environment variables, + or configuration file (CLI takes highest precedence). + + Args: + parser: ArgumentParser to add arguments to + """ + parser.add_argument( + "--namespace", + help="OpenLineage namespace", + ) + parser.add_argument( + "--job-name", + help="Job name", + ) + parser.add_argument( + "--marquez-url", + help="Marquez server URL", + ) + parser.add_argument( + "--facet-prefix", + help="Facet prefix (default: empty string)", + ) + + +def _add_common_facet_args(parser: argparse.ArgumentParser) -> None: + """ + Add CommonRunFacet arguments. + + These arguments represent robot and repository metadata required + for OpenLineage event tracking. + + Args: + parser: ArgumentParser to add arguments to + """ + parser.add_argument( + "--robot-id", + help="Robot identifier", + ) + parser.add_argument( + "--location", + help="Location identifier", + ) + parser.add_argument( + "--repository-hash", + help="Git commit hash", + ) + parser.add_argument( + "--repository-uri", + help="Repository URI", + ) + parser.add_argument( + "--repository-tag", + help="Git tag", + ) + parser.add_argument( + "--repository-branch", + help="Git branch", + ) + + +def create_parser() -> argparse.ArgumentParser: + """ + Create argument parser for airoa-lineage-wasabi-upload CLI. + + Returns: + Configured ArgumentParser instance + """ + parser = argparse.ArgumentParser( + prog="airoa-lineage-wasabi-upload", + description="Track Wasabi upload operations with OpenLineage", + ) + + parser.add_argument( + "--version", + action="version", + version="airoa-lineage-wasabi-upload 0.1.0", + ) + + parser.add_argument( + "--config", + type=str, + help="Path to config file (default: ~/.config/airoa-lineage/config.json)", + ) + + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Enable verbose output", + ) + + parser.add_argument( + "-q", + "--quiet", + action="store_true", + help="Suppress output (except errors)", + ) + + parser.add_argument( + "--json", + action="store_true", + help="Output in JSON format", + ) + + # Create subparsers for commands + subparsers = parser.add_subparsers(dest="command", required=True) + + # START command + start_parser = subparsers.add_parser( + "start", + help="Start Wasabi upload session and emit START event", + ) + _add_common_args(start_parser) + _add_common_facet_args(start_parser) + start_parser.add_argument( + "--nominal-start-time", + help="Nominal start time (ISO 8601 format)", + ) + start_parser.add_argument( + "--nominal-end-time", + help="Nominal end time (ISO 8601 format)", + ) + start_parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be done without executing", + ) + + # COMPLETE command + complete_parser = subparsers.add_parser( + "complete", + help="Complete Wasabi upload session and emit COMPLETE event", + ) + complete_parser.add_argument( + "--run-id", + required=True, + help="Session run ID (from start command)", + ) + _add_common_args(complete_parser) + _add_common_facet_args(complete_parser) + + # CANCEL command + cancel_parser = subparsers.add_parser( + "cancel", + help="Cancel Wasabi upload session and emit ABORT event", + ) + cancel_parser.add_argument( + "--run-id", + required=True, + help="Session run ID (from start command)", + ) + _add_common_args(cancel_parser) + _add_common_facet_args(cancel_parser) + + return parser + + +def merge_config_with_args( + config: Dict[str, Any], + args: argparse.Namespace, +) -> Dict[str, Any]: + """ + Merge configuration with CLI arguments. + + CLI arguments take precedence over config file and environment variables. + + Args: + config: Configuration from load_config() + args: Parsed CLI arguments + + Returns: + Merged configuration dictionary + """ + # Top-level configuration + if args.namespace: + config["namespace"] = args.namespace + + if hasattr(args, "marquez_url") and args.marquez_url: + config["marquez_url"] = args.marquez_url + + if hasattr(args, "job_name") and args.job_name: + config["job_name"] = args.job_name + + if hasattr(args, "facet_prefix") and args.facet_prefix: + config["facet_prefix"] = args.facet_prefix + + # CommonRunFacet fields + if hasattr(args, "robot_id") and args.robot_id: + if "common_facet" not in config: + config["common_facet"] = {} + config["common_facet"]["robotId"] = args.robot_id + + if hasattr(args, "location") and args.location: + if "common_facet" not in config: + config["common_facet"] = {} + config["common_facet"]["location"] = args.location + + if hasattr(args, "repository_hash") and args.repository_hash: + if "common_facet" not in config: + config["common_facet"] = {} + config["common_facet"]["repositoryHash"] = args.repository_hash + + if hasattr(args, "repository_uri") and args.repository_uri: + if "common_facet" not in config: + config["common_facet"] = {} + config["common_facet"]["repositoryUri"] = args.repository_uri + + if hasattr(args, "repository_tag") and args.repository_tag: + if "common_facet" not in config: + config["common_facet"] = {} + config["common_facet"]["repositoryTag"] = args.repository_tag + + if hasattr(args, "repository_branch") and args.repository_branch: + if "common_facet" not in config: + config["common_facet"] = {} + config["common_facet"]["repositoryBranch"] = args.repository_branch + + return config + + +def cmd_start(args: argparse.Namespace, config: Dict[str, Any]) -> int: + """ + Execute start command. + + Args: + args: Parsed CLI arguments + config: Merged configuration + + Returns: + Exit code (0=success, 1=error) + """ + try: + # Build CommonRunFacet + common_facet = build_common_facet(config) + + # Validate required fields + if "namespace" not in config: + print( + "Error: namespace is required. " + "Provide via --namespace, config file, or AIROA_NAMESPACE env var.", + file=sys.stderr, + ) + return 1 + + # Dry run mode + if args.dry_run: + print("[DRY RUN] Would start session with:") + print(f" Namespace: {config['namespace']}") + print(f" Job name: {config.get('job_name', 'wasabi-data-upload')}") + print(f" Robot ID: {common_facet.robotId}") + print(f" Location: {common_facet.location}") + print(f" Repository: {common_facet.repositoryUri}") + print(f" Commit: {common_facet.repositoryHash}") + print(f" Branch: {common_facet.repositoryBranch}") + print(f" Tag: {common_facet.repositoryTag}") + if hasattr(args, "nominal_start_time") and args.nominal_start_time: + print(f" Nominal start: {args.nominal_start_time}") + if hasattr(args, "nominal_end_time") and args.nominal_end_time: + print(f" Nominal end: {args.nominal_end_time}") + print("[DRY RUN] Would output run_id (not generated in dry-run mode)") + return 0 + + # Create session + session = WasabiUploadSession( + namespace=config["namespace"], + common_facet=common_facet, + job_name=config.get("job_name", "wasabi-data-upload"), + marquez_url=config.get("marquez_url"), + facet_prefix=config.get("facet_prefix", ""), + ) + + # Start session + run_id = session.start( + nominal_start_time=getattr(args, "nominal_start_time", None), + nominal_end_time=getattr(args, "nominal_end_time", None), + ) + + # Output run_id + if args.json: + output = {"run_id": run_id, "status": "started"} + print(json.dumps(output)) + elif args.verbose: + print("Session started successfully", file=sys.stderr) + print(run_id) + else: + # Default: just print run_id + print(run_id) + + return 0 + + except ValueError as e: + print(f"Configuration error: {e}", file=sys.stderr) + return 3 + except RuntimeError as e: + print(f"Session error: {e}", file=sys.stderr) + return 1 + except Exception as e: + print(f"Unexpected error: {e}", file=sys.stderr) + if args.verbose: + import traceback + + traceback.print_exc() + return 1 + + +def cmd_complete(args: argparse.Namespace, config: Dict[str, Any]) -> int: + """ + Execute complete command. + + Args: + args: Parsed CLI arguments + config: Merged configuration + + Returns: + Exit code (0=success, 1=error) + """ + try: + # Build CommonRunFacet + common_facet = build_common_facet(config) + + # Validate required fields + if "namespace" not in config: + print( + "Error: namespace is required. " + "Provide via --namespace, config file, or AIROA_NAMESPACE env var.", + file=sys.stderr, + ) + return 1 + + # Create session + session = WasabiUploadSession( + namespace=config["namespace"], + common_facet=common_facet, + job_name=config.get("job_name", "wasabi-data-upload"), + marquez_url=config.get("marquez_url"), + facet_prefix=config.get("facet_prefix", ""), + ) + + # Resume existing session + session.resume(args.run_id) + + # Complete session + session.complete() + + # Output success message + if not args.quiet: + if args.json: + output = {"run_id": args.run_id, "status": "completed"} + print(json.dumps(output)) + else: + print(f"Session {args.run_id} completed successfully") + + return 0 + + except ValueError as e: + print(f"Configuration error: {e}", file=sys.stderr) + return 3 + except RuntimeError as e: + print(f"Session error: {e}", file=sys.stderr) + return 1 + except Exception as e: + print(f"Unexpected error: {e}", file=sys.stderr) + if args.verbose: + import traceback + + traceback.print_exc() + return 1 + + +def cmd_cancel(args: argparse.Namespace, config: Dict[str, Any]) -> int: + """ + Execute cancel command. + + Args: + args: Parsed CLI arguments + config: Merged configuration + + Returns: + Exit code (0=success, 1=error) + """ + try: + # Build CommonRunFacet + common_facet = build_common_facet(config) + + # Validate required fields + if "namespace" not in config: + print( + "Error: namespace is required. " + "Provide via --namespace, config file, or AIROA_NAMESPACE env var.", + file=sys.stderr, + ) + return 1 + + # Create session + session = WasabiUploadSession( + namespace=config["namespace"], + common_facet=common_facet, + job_name=config.get("job_name", "wasabi-data-upload"), + marquez_url=config.get("marquez_url"), + facet_prefix=config.get("facet_prefix", ""), + ) + + # Resume existing session + session.resume(args.run_id) + + # Cancel session + session.cancel() + + # Output success message + if not args.quiet: + if args.json: + output = {"run_id": args.run_id, "status": "cancelled"} + print(json.dumps(output)) + else: + print(f"Session {args.run_id} cancelled successfully") + + return 0 + + except ValueError as e: + print(f"Configuration error: {e}", file=sys.stderr) + return 3 + except RuntimeError as e: + print(f"Session error: {e}", file=sys.stderr) + return 1 + except Exception as e: + print(f"Unexpected error: {e}", file=sys.stderr) + if args.verbose: + import traceback + + traceback.print_exc() + return 1 + + +def main() -> int: + """ + Main entry point for airoa-lineage-wasabi-upload CLI. + + Returns: + Exit code (0=success, 1=general error, 2=connection error, 3=config error) + """ + parser = create_parser() + args = parser.parse_args() + + # Load configuration + try: + config_path = Path(args.config) if args.config else None + config = load_config(config_path) + except FileNotFoundError as e: + print(f"Configuration file not found: {e}", file=sys.stderr) + return 3 + except json.JSONDecodeError as e: + print(f"Invalid JSON in configuration file: {e}", file=sys.stderr) + return 3 + except Exception as e: + print(f"Error loading configuration: {e}", file=sys.stderr) + return 3 + + # Merge with CLI arguments + config = merge_config_with_args(config, args) + + # Execute command + if args.command == "start": + return cmd_start(args, config) + elif args.command == "complete": + return cmd_complete(args, config) + elif args.command == "cancel": + return cmd_cancel(args, config) + else: + print(f"Unknown command: {args.command}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/cli/test_wasabi_upload.py b/tests/unit/cli/test_wasabi_upload.py new file mode 100644 index 0000000..9359898 --- /dev/null +++ b/tests/unit/cli/test_wasabi_upload.py @@ -0,0 +1,268 @@ +"""Tests for CLI wasabi_upload module.""" + +import argparse + + +from airoa_lineage.cli.wasabi_upload import ( + _add_common_args, + _add_common_facet_args, + create_parser, + merge_config_with_args, +) + + +class TestHelperFunctions: + """Test helper functions for argument parsing.""" + + def test_add_common_args(self): + """Test that _add_common_args adds expected arguments.""" + parser = argparse.ArgumentParser() + _add_common_args(parser) + + # Parse with arguments + args = parser.parse_args( + [ + "--namespace", + "test", + "--job-name", + "test-job", + "--marquez-url", + "http://localhost:9000", + "--facet-prefix", + "test-prefix", + ] + ) + + assert args.namespace == "test" + assert args.job_name == "test-job" + assert args.marquez_url == "http://localhost:9000" + assert args.facet_prefix == "test-prefix" + + def test_add_common_facet_args(self): + """Test that _add_common_facet_args adds expected arguments.""" + parser = argparse.ArgumentParser() + _add_common_facet_args(parser) + + # Parse with arguments + args = parser.parse_args( + [ + "--robot-id", + "hsr001", + "--location", + "weblab", + "--repository-hash", + "abc123", + "--repository-uri", + "https://github.com/test/repo.git", + "--repository-tag", + "v1.0.0", + "--repository-branch", + "main", + ] + ) + + assert args.robot_id == "hsr001" + assert args.location == "weblab" + assert args.repository_hash == "abc123" + assert args.repository_uri == "https://github.com/test/repo.git" + assert args.repository_tag == "v1.0.0" + assert args.repository_branch == "main" + + +class TestCreateParser: + """Test argument parser creation.""" + + def test_complete_parser_has_common_facet_args(self): + """Test that complete command has CommonRunFacet arguments.""" + parser = create_parser() + + # Parse complete command with CommonRunFacet args + args = parser.parse_args( + [ + "complete", + "--run-id", + "550e8400-e29b-41d4-a716-446655440000", + "--robot-id", + "hsr001", + "--location", + "weblab", + "--repository-hash", + "abc123", + "--repository-uri", + "https://github.com/test/repo.git", + "--repository-tag", + "v1.0.0", + "--repository-branch", + "main", + ] + ) + + assert args.command == "complete" + assert args.run_id == "550e8400-e29b-41d4-a716-446655440000" + assert args.robot_id == "hsr001" + assert args.location == "weblab" + assert args.repository_hash == "abc123" + assert args.repository_uri == "https://github.com/test/repo.git" + assert args.repository_tag == "v1.0.0" + assert args.repository_branch == "main" + + def test_complete_parser_has_facet_prefix(self): + """Test that complete command has --facet-prefix argument.""" + parser = create_parser() + + args = parser.parse_args( + [ + "complete", + "--run-id", + "550e8400-e29b-41d4-a716-446655440000", + "--facet-prefix", + "airoa", + ] + ) + + assert args.facet_prefix == "airoa" + + def test_cancel_parser_has_common_facet_args(self): + """Test that cancel command has CommonRunFacet arguments.""" + parser = create_parser() + + # Parse cancel command with CommonRunFacet args + args = parser.parse_args( + [ + "cancel", + "--run-id", + "550e8400-e29b-41d4-a716-446655440000", + "--robot-id", + "hsr001", + "--location", + "weblab", + "--repository-hash", + "abc123", + "--repository-uri", + "https://github.com/test/repo.git", + "--repository-tag", + "v1.0.0", + "--repository-branch", + "main", + ] + ) + + assert args.command == "cancel" + assert args.run_id == "550e8400-e29b-41d4-a716-446655440000" + assert args.robot_id == "hsr001" + assert args.location == "weblab" + + def test_cancel_parser_has_facet_prefix(self): + """Test that cancel command has --facet-prefix argument.""" + parser = create_parser() + + args = parser.parse_args( + [ + "cancel", + "--run-id", + "550e8400-e29b-41d4-a716-446655440000", + "--facet-prefix", + "airoa", + ] + ) + + assert args.facet_prefix == "airoa" + + +class TestMergeConfigWithArgs: + """Test configuration merging with CLI arguments.""" + + def test_complete_cli_args_override_config(self): + """Test that complete CLI args override config values.""" + config = { + "namespace": "config-namespace", + "common_facet": { + "robotId": "config-robot", + "location": "config-location", + }, + } + + # Simulate complete command args + args = argparse.Namespace( + command="complete", + run_id="550e8400-e29b-41d4-a716-446655440000", + namespace="cli-namespace", + robot_id="cli-robot", + location="cli-location", + repository_hash="cli-hash", + repository_uri="https://github.com/test/repo.git", + repository_tag="v1.0.0", + repository_branch="main", + job_name=None, + marquez_url=None, + facet_prefix=None, + ) + + result = merge_config_with_args(config, args) + + assert result["namespace"] == "cli-namespace" + assert result["common_facet"]["robotId"] == "cli-robot" + assert result["common_facet"]["location"] == "cli-location" + assert result["common_facet"]["repositoryHash"] == "cli-hash" + assert ( + result["common_facet"]["repositoryUri"] + == "https://github.com/test/repo.git" + ) + assert result["common_facet"]["repositoryTag"] == "v1.0.0" + assert result["common_facet"]["repositoryBranch"] == "main" + + def test_facet_prefix_merge_in_complete(self): + """Test that --facet-prefix is correctly merged in complete.""" + config = {"facet_prefix": "config-prefix"} + + args = argparse.Namespace( + command="complete", + run_id="550e8400-e29b-41d4-a716-446655440000", + namespace=None, + job_name=None, + marquez_url=None, + facet_prefix="cli-prefix", + robot_id=None, + location=None, + repository_hash=None, + repository_uri=None, + repository_tag=None, + repository_branch=None, + ) + + result = merge_config_with_args(config, args) + + assert result["facet_prefix"] == "cli-prefix" + + def test_cancel_cli_args_override_config(self): + """Test that cancel CLI args override config values.""" + config = { + "namespace": "config-namespace", + "common_facet": { + "robotId": "config-robot", + "location": "config-location", + }, + } + + # Simulate cancel command args + args = argparse.Namespace( + command="cancel", + run_id="550e8400-e29b-41d4-a716-446655440000", + namespace="cli-namespace", + robot_id="cli-robot", + location="cli-location", + repository_hash="cli-hash", + repository_uri="https://github.com/test/repo.git", + repository_tag="v1.0.0", + repository_branch="main", + job_name=None, + marquez_url=None, + facet_prefix=None, + ) + + result = merge_config_with_args(config, args) + + assert result["namespace"] == "cli-namespace" + assert result["common_facet"]["robotId"] == "cli-robot" + assert result["common_facet"]["location"] == "cli-location" + assert result["common_facet"]["repositoryHash"] == "cli-hash"