From 44323d4207377d5b455af44b907baba1e04b25c7 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 21 Mar 2026 13:13:40 +0100 Subject: [PATCH 1/2] Add optional CLI for downstream software in mache.deploy This will allow Compass to add a `--with-albany` flag, for example, and use it to optionally install the Albany and Trilinos libraries. --- mache/deploy/cli.py | 4 +- mache/deploy/cli_spec.py | 89 +++++++++++++++---- mache/deploy/init_update.py | 10 +++ mache/deploy/templates/custom_cli_spec.json | 3 + mache/deploy/templates/deploy.py.j2 | 74 ++++++++++++++- tests/test_deploy_cli_spec.py | 53 +++++++++++ tests/test_deploy_init_update.py | 54 +++++++++++ .../deploy_hook_overrides/config.yaml.j2 | 26 ++++++ .../custom_cli_spec.json | 11 +++ .../fixtures/deploy_hook_overrides/hooks.py | 7 ++ workflow_tests/helpers.py | 11 ++- workflow_tests/test_deploy_workflow.py | 11 +++ 12 files changed, 332 insertions(+), 21 deletions(-) create mode 100644 mache/deploy/templates/custom_cli_spec.json create mode 100644 tests/test_deploy_cli_spec.py create mode 100644 tests/test_deploy_init_update.py create mode 100644 workflow_tests/fixtures/deploy_hook_overrides/config.yaml.j2 create mode 100644 workflow_tests/fixtures/deploy_hook_overrides/custom_cli_spec.json create mode 100644 workflow_tests/fixtures/deploy_hook_overrides/hooks.py diff --git a/mache/deploy/cli.py b/mache/deploy/cli.py index cb54ba76..a70a30b6 100644 --- a/mache/deploy/cli.py +++ b/mache/deploy/cli.py @@ -6,7 +6,7 @@ from .cli_spec import ( add_args_to_parser, filter_args_by_route, - load_cli_spec_file, + load_repo_cli_spec_file, ) from .init_update import init_or_update_repo from .run import run_deploy @@ -57,7 +57,7 @@ def add_deploy_subparser(subparsers: argparse._SubParsersAction) -> None: 'deploy.py)', ) - spec = load_cli_spec_file() + spec = load_repo_cli_spec_file() run_args = filter_args_by_route(spec, 'run') add_args_to_parser(p_run, run_args) diff --git a/mache/deploy/cli_spec.py b/mache/deploy/cli_spec.py index 8bf7ce80..ddca0965 100644 --- a/mache/deploy/cli_spec.py +++ b/mache/deploy/cli_spec.py @@ -4,6 +4,7 @@ import json from dataclasses import dataclass from importlib import resources +from pathlib import Path from typing import Any @@ -21,32 +22,43 @@ class CliSpec: args: list[CliArgSpec] -def parse_cli_spec(rendered_json: str) -> CliSpec: +def parse_cli_spec( + rendered_json: str, + *, + source: str = 'cli_spec', + require_meta: bool = True, +) -> CliSpec: try: spec = json.loads(rendered_json) except json.JSONDecodeError as e: - raise ValueError(f'cli_spec rendered to invalid JSON: {e}') from e + raise ValueError(f'{source} rendered to invalid JSON: {e}') from e if not isinstance(spec, dict): - raise ValueError('cli_spec must be a JSON object') + raise ValueError(f'{source} must be a JSON object') - meta = spec.get('meta') args = spec.get('arguments') - - if not isinstance(meta, dict) or not isinstance(args, list): - raise ValueError( - "cli_spec must contain object 'meta' and list 'arguments'" - ) - - if 'mache_version' not in meta or not str(meta['mache_version']).strip(): + meta = spec.get('meta', {}) + + if not isinstance(args, list): + raise ValueError(f"{source} must contain list 'arguments'") + if require_meta and not isinstance(meta, dict): + raise ValueError(f"{source} must contain object 'meta'") + if not require_meta and meta is None: + meta = {} + if not isinstance(meta, dict): + raise ValueError(f'{source}.meta must be an object') + + if require_meta and ( + 'mache_version' not in meta or not str(meta['mache_version']).strip() + ): raise ValueError( - 'cli_spec.meta.mache_version is required and must be non-empty' + f'{source}.meta.mache_version is required and must be non-empty' ) parsed_args: list[CliArgSpec] = [] for i, entry in enumerate(args): if not isinstance(entry, dict): - raise ValueError(f'cli_spec.arguments[{i}] must be an object') + raise ValueError(f'{source}.arguments[{i}] must be an object') flags = entry.get('flags') dest = entry.get('dest') @@ -58,18 +70,18 @@ def parse_cli_spec(rendered_json: str) -> CliSpec: or not flags ): raise ValueError( - f'cli_spec.arguments[{i}].flags must be a non-empty list[str]' + f'{source}.arguments[{i}].flags must be a non-empty list[str]' ) if not isinstance(dest, str) or not dest.strip(): raise ValueError( - f'cli_spec.arguments[{i}].dest must be a non-empty string' + f'{source}.arguments[{i}].dest must be a non-empty string' ) if isinstance(route, list) and all(isinstance(r, str) for r in route): route_list = route else: raise ValueError( - f'cli_spec.arguments[{i}].route must be a list[str]' + f'{source}.arguments[{i}].route must be a list[str]' ) # Allow only a safe subset of argparse kwargs @@ -92,6 +104,34 @@ def parse_cli_spec(rendered_json: str) -> CliSpec: return CliSpec(meta=meta, args=parsed_args) +def merge_cli_specs(base: CliSpec, extra: CliSpec | None) -> CliSpec: + if extra is None: + return base + + merged_args = list(base.args) + seen_dests = {arg.dest for arg in base.args} + seen_flags = {flag for arg in base.args for flag in arg.flags} + + for arg in extra.args: + if arg.dest in seen_dests: + raise ValueError( + f'custom_cli_spec dest duplicates generated cli_spec: ' + f'{arg.dest}' + ) + duplicate_flags = [flag for flag in arg.flags if flag in seen_flags] + if duplicate_flags: + dup_str = ', '.join(duplicate_flags) + raise ValueError( + f'custom_cli_spec flags duplicate generated cli_spec: ' + f'{dup_str}' + ) + merged_args.append(arg) + seen_dests.add(arg.dest) + seen_flags.update(arg.flags) + + return CliSpec(meta=dict(base.meta), args=merged_args) + + def routes_include(arg: CliArgSpec, route: str) -> bool: return route in arg.route @@ -104,7 +144,22 @@ def load_cli_spec_file() -> CliSpec: with resources.open_text( 'mache.deploy.templates', 'cli_spec.json.j2' ) as f: - return parse_cli_spec(f.read()) + return parse_cli_spec(f.read(), source='cli_spec') + + +def load_repo_cli_spec_file(repo_root: str = '.') -> CliSpec: + spec = load_cli_spec_file() + custom_spec_path = Path(repo_root) / 'deploy' / 'custom_cli_spec.json' + if not custom_spec_path.exists(): + return spec + + custom_text = custom_spec_path.read_text(encoding='utf-8') + custom_spec = parse_cli_spec( + custom_text, + source=str(custom_spec_path), + require_meta=False, + ) + return merge_cli_specs(spec, custom_spec) def add_args_to_parser( diff --git a/mache/deploy/init_update.py b/mache/deploy/init_update.py index f146953a..3fc97d85 100644 --- a/mache/deploy/init_update.py +++ b/mache/deploy/init_update.py @@ -27,6 +27,7 @@ def init_or_update_repo( Writes: - deploy.py (repo root) - deploy/cli_spec.json + - deploy/custom_cli_spec.json (init only, downstream-owned) - deploy/pins.cfg (init only) - deploy/config.yaml.j2 (init only) - deploy/spack.yaml.j2 (init only) @@ -146,6 +147,15 @@ def init_or_update_repo( overwrite=overwrite, ) + custom_cli_spec_tmpl = _read_pkg_template( + f'{TEMPLATE_DIR}/custom_cli_spec.json' + ) + _write_text( + deploy_dir / 'custom_cli_spec.json', + custom_cli_spec_tmpl, + overwrite=overwrite, + ) + load_sh = ( f'# bash snippet for adding {software}-specific environment ' f'variables\n' diff --git a/mache/deploy/templates/custom_cli_spec.json b/mache/deploy/templates/custom_cli_spec.json new file mode 100644 index 00000000..ed8b61bd --- /dev/null +++ b/mache/deploy/templates/custom_cli_spec.json @@ -0,0 +1,3 @@ +{ + "arguments": [] +} diff --git a/mache/deploy/templates/deploy.py.j2 b/mache/deploy/templates/deploy.py.j2 index b16c7503..369d0d3e 100755 --- a/mache/deploy/templates/deploy.py.j2 +++ b/mache/deploy/templates/deploy.py.j2 @@ -3,7 +3,8 @@ Target software deployment entrypoint. - Reads pinned mache version from deploy/pins.cfg -- Reads CLI spec from deploy/cli_spec.json and builds argparse CLI +- Reads CLI spec from deploy/cli_spec.json plus optional + deploy/custom_cli_spec.json and builds argparse CLI - Downloads mache/deploy/bootstrap.py for either: * a given mache fork/branch, or * the pinned mache version @@ -25,6 +26,7 @@ from urllib.request import Request, urlopen PINS_CFG = os.path.join('deploy', 'pins.cfg') CLI_SPEC_JSON = os.path.join('deploy', 'cli_spec.json') +CUSTOM_CLI_SPEC_JSON = os.path.join('deploy', 'custom_cli_spec.json') DEPLOY_TMP_DIR = 'deploy_tmp' BOOTSTRAP_PATH = os.path.join(DEPLOY_TMP_DIR, 'bootstrap.py') @@ -40,6 +42,10 @@ def main(): pinned_mache_version, pinned_python_version = _read_pins(PINS_CFG) cli_spec = _read_cli_spec(CLI_SPEC_JSON) + cli_spec = _merge_optional_cli_spec( + cli_spec, + _read_optional_cli_spec(CUSTOM_CLI_SPEC_JSON), + ) parser = _build_parser_from_cli_spec(cli_spec) args = parser.parse_args(sys.argv[1:]) @@ -225,6 +231,72 @@ def _read_cli_spec(spec_path): return spec +def _read_optional_cli_spec(spec_path): + if not os.path.exists(spec_path): + return None + + try: + with open(spec_path, 'r', encoding='utf-8') as f: + spec = json.load(f) + except (OSError, json.JSONDecodeError) as e: + raise SystemExit(f'ERROR: Failed to parse {spec_path}: {e!r}') from e + + if not isinstance(spec, dict): + raise SystemExit(f'ERROR: {spec_path} must contain a JSON object') + if 'arguments' not in spec: + raise SystemExit( + f"ERROR: {spec_path} must contain top-level key 'arguments'" + ) + if not isinstance(spec['arguments'], list): + raise SystemExit(f"ERROR: {spec_path} 'arguments' must be a list") + meta = spec.get('meta') + if meta is not None and not isinstance(meta, dict): + raise SystemExit(f"ERROR: {spec_path} 'meta' must be an object") + + return spec + + +def _merge_optional_cli_spec(cli_spec, custom_cli_spec): + if custom_cli_spec is None: + return cli_spec + + merged = { + 'meta': dict(cli_spec.get('meta', {})), + 'arguments': list(cli_spec.get('arguments', [])), + } + + seen_dests = set() + seen_flags = set() + for entry in merged['arguments']: + dest = entry.get('dest') + if dest: + seen_dests.add(dest) + for flag in entry.get('flags', []): + seen_flags.add(flag) + + for entry in custom_cli_spec['arguments']: + dest = entry.get('dest') + if dest in seen_dests: + raise SystemExit( + 'ERROR: deploy/custom_cli_spec.json duplicates generated ' + f"dest '{dest}'" + ) + flags = entry.get('flags', []) + duplicate_flags = [flag for flag in flags if flag in seen_flags] + if duplicate_flags: + dup_str = ', '.join(duplicate_flags) + raise SystemExit( + 'ERROR: deploy/custom_cli_spec.json duplicates generated ' + f'flags: {dup_str}' + ) + merged['arguments'].append(entry) + if dest: + seen_dests.add(dest) + seen_flags.update(flags) + + return merged + + def _build_parser_from_cli_spec(cli_spec): description = cli_spec.get('meta', {}).get( 'description', 'Deploy E3SM software environment' diff --git a/tests/test_deploy_cli_spec.py b/tests/test_deploy_cli_spec.py new file mode 100644 index 00000000..dc297c63 --- /dev/null +++ b/tests/test_deploy_cli_spec.py @@ -0,0 +1,53 @@ +from pathlib import Path + +import pytest + +from mache.deploy.cli_spec import filter_args_by_route, load_repo_cli_spec_file + + +def test_load_repo_cli_spec_file_merges_custom_flags(tmp_path: Path): + deploy_dir = tmp_path / 'deploy' + deploy_dir.mkdir() + (deploy_dir / 'custom_cli_spec.json').write_text( + '{\n' + ' "arguments": [\n' + ' {\n' + ' "flags": ["--with-albany"],\n' + ' "dest": "with_albany",\n' + ' "action": "store_true",\n' + ' "help": "Install optional Albany support.",\n' + ' "route": ["deploy", "run"]\n' + ' }\n' + ' ]\n' + '}\n', + encoding='utf-8', + ) + + spec = load_repo_cli_spec_file(str(tmp_path)) + run_args = filter_args_by_route(spec, 'run') + + assert any(arg.dest == 'with_albany' for arg in run_args) + + +def test_load_repo_cli_spec_file_rejects_duplicate_custom_flags( + tmp_path: Path, +): + deploy_dir = tmp_path / 'deploy' + deploy_dir.mkdir() + (deploy_dir / 'custom_cli_spec.json').write_text( + '{\n' + ' "arguments": [\n' + ' {\n' + ' "flags": ["--machine"],\n' + ' "dest": "with_albany",\n' + ' "route": ["deploy", "run"]\n' + ' }\n' + ' ]\n' + '}\n', + encoding='utf-8', + ) + + with pytest.raises( + ValueError, match='custom_cli_spec flags duplicate generated cli_spec' + ): + load_repo_cli_spec_file(str(tmp_path)) diff --git a/tests/test_deploy_init_update.py b/tests/test_deploy_init_update.py new file mode 100644 index 00000000..324336ce --- /dev/null +++ b/tests/test_deploy_init_update.py @@ -0,0 +1,54 @@ +from pathlib import Path + +from mache.deploy.init_update import init_or_update_repo + + +def test_init_creates_custom_cli_spec_file(tmp_path: Path): + init_or_update_repo( + repo_root=str(tmp_path), + software='polaris', + mache_version='1.2.3', + update=False, + overwrite=False, + ) + + custom_cli_spec = tmp_path / 'deploy' / 'custom_cli_spec.json' + assert custom_cli_spec.read_text(encoding='utf-8') == ( + '{\n "arguments": []\n}\n' + ) + + +def test_update_preserves_custom_cli_spec_file(tmp_path: Path): + init_or_update_repo( + repo_root=str(tmp_path), + software='polaris', + mache_version='1.2.3', + update=False, + overwrite=False, + ) + + custom_cli_spec = tmp_path / 'deploy' / 'custom_cli_spec.json' + expected = ( + '{\n' + ' "arguments": [\n' + ' {\n' + ' "flags": ["--with-albany"],\n' + ' "dest": "with_albany",\n' + ' "action": "store_true",\n' + ' "help": "Install optional Albany support.",\n' + ' "route": ["deploy", "run"]\n' + ' }\n' + ' ]\n' + '}\n' + ) + custom_cli_spec.write_text(expected, encoding='utf-8') + + init_or_update_repo( + repo_root=str(tmp_path), + software='polaris', + mache_version='1.2.4', + update=True, + overwrite=True, + ) + + assert custom_cli_spec.read_text(encoding='utf-8') == expected diff --git a/workflow_tests/fixtures/deploy_hook_overrides/config.yaml.j2 b/workflow_tests/fixtures/deploy_hook_overrides/config.yaml.j2 new file mode 100644 index 00000000..6ad547ef --- /dev/null +++ b/workflow_tests/fixtures/deploy_hook_overrides/config.yaml.j2 @@ -0,0 +1,26 @@ +project: + software: "toyflow" + version: "0.1.0" + machine: null + runtime_version_cmd: >- + python -c 'from toyflow.version import __version__; print(__version__)' + +pixi: + deploy: true + prefix: "pixi-env" + channels: + - conda-forge + mpi: "nompi" + install_dev_software: true + +spack: + deploy: false + supported: false + +jigsaw: + enabled: false + +hooks: + file: "deploy/hooks.py" + entrypoints: + post_pixi: "post_pixi" diff --git a/workflow_tests/fixtures/deploy_hook_overrides/custom_cli_spec.json b/workflow_tests/fixtures/deploy_hook_overrides/custom_cli_spec.json new file mode 100644 index 00000000..a4999c70 --- /dev/null +++ b/workflow_tests/fixtures/deploy_hook_overrides/custom_cli_spec.json @@ -0,0 +1,11 @@ +{ + "arguments": [ + { + "flags": ["--with-albany"], + "dest": "with_albany", + "action": "store_true", + "help": "Install optional Albany support.", + "route": ["deploy", "run"] + } + ] +} diff --git a/workflow_tests/fixtures/deploy_hook_overrides/hooks.py b/workflow_tests/fixtures/deploy_hook_overrides/hooks.py new file mode 100644 index 00000000..aa3250df --- /dev/null +++ b/workflow_tests/fixtures/deploy_hook_overrides/hooks.py @@ -0,0 +1,7 @@ +from pathlib import Path + + +def post_pixi(ctx): + if getattr(ctx.args, 'with_albany', False): + marker = Path(ctx.work_dir) / 'with_albany.txt' + marker.write_text('enabled\n', encoding='utf-8') diff --git a/workflow_tests/helpers.py b/workflow_tests/helpers.py index 665697e8..7d2b6b90 100644 --- a/workflow_tests/helpers.py +++ b/workflow_tests/helpers.py @@ -29,10 +29,19 @@ def copy_fixture_repo(fixture_name: str, dest: Path) -> None: def configure_generated_deploy_files( repo_root: Path, overrides_name: str +) -> dict[str, str]: + return configure_deploy_files( + repo_root, + overrides_name, + relpaths=('config.yaml.j2', 'pixi.toml.j2', 'load.sh'), + ) + + +def configure_deploy_files( + repo_root: Path, overrides_name: str, relpaths: tuple[str, ...] ) -> dict[str, str]: deploy_dir = repo_root / 'deploy' overrides_dir = fixtures_dir() / overrides_name - relpaths = ('config.yaml.j2', 'pixi.toml.j2', 'load.sh') expected: dict[str, str] = {} for relpath in relpaths: diff --git a/workflow_tests/test_deploy_workflow.py b/workflow_tests/test_deploy_workflow.py index dcdb402e..3eed2f06 100644 --- a/workflow_tests/test_deploy_workflow.py +++ b/workflow_tests/test_deploy_workflow.py @@ -6,6 +6,7 @@ import pytest from workflow_tests.helpers import ( + configure_deploy_files, configure_generated_deploy_files, copy_fixture_repo, init_and_update_repo, @@ -52,6 +53,12 @@ def test_downstream_deploy_workflow(tmp_path: Path): encoding='utf-8' ) == expected + configure_deploy_files( + downstream, + 'deploy_hook_overrides', + relpaths=('config.yaml.j2', 'custom_cli_spec.json', 'hooks.py'), + ) + run( [ str(downstream / 'deploy.py'), @@ -63,6 +70,7 @@ def test_downstream_deploy_workflow(tmp_path: Path): 'local/mache', '--mache-branch', 'current', + '--with-albany', '--recreate', ], cwd=downstream, @@ -71,6 +79,9 @@ def test_downstream_deploy_workflow(tmp_path: Path): load_script = downstream / 'load_toyflow.sh' assert load_script.is_file() + assert (downstream / 'deploy_tmp' / 'with_albany.txt').read_text( + encoding='utf-8' + ) == 'enabled\n' smoke = run( [ From 64a4e3187f7006128e3d6f689eac071107c9a9df Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 21 Mar 2026 13:27:48 +0100 Subject: [PATCH 2/2] Document the new custom CLI capability --- docs/design/mache_deploy.md | 21 ++- docs/developers_guide/api.md | 2 + docs/developers_guide/deploy.md | 16 +- docs/users_guide/deploy.md | 140 +++++++++++++++++- mache/deploy/templates/config.yaml.j2.j2 | 2 + mache/deploy/templates/hooks.py.j2 | 30 ++++ .../custom_cli_spec.json | 6 + .../fixtures/deploy_hook_overrides/hooks.py | 5 + workflow_tests/test_deploy_workflow.py | 5 + 9 files changed, 211 insertions(+), 16 deletions(-) diff --git a/docs/design/mache_deploy.md b/docs/design/mache_deploy.md index 2f86502a..68e5e9e6 100644 --- a/docs/design/mache_deploy.md +++ b/docs/design/mache_deploy.md @@ -294,7 +294,8 @@ This separation is intentional and strictly enforced. ### Design resolution: `cli_spec.json(.j2)` Each target software includes `deploy/cli_spec.json` rendered from the -packaged template, which declaratively defines: +packaged template, plus an optional downstream-owned +`deploy/custom_cli_spec.json`, which declaratively define: - Command-line flags - Help text @@ -304,17 +305,21 @@ packaged template, which declaratively defines: **Usage** - `deploy.py` - - Builds its argparse interface from `deploy/cli_spec.json` + - Builds its argparse interface from `deploy/cli_spec.json` plus optional + `deploy/custom_cli_spec.json` - Forwards appropriate arguments to `bootstrap.py` and `mache deploy run` - `mache deploy run` - Builds its CLI from the packaged `mache/deploy/templates/cli_spec.json.j2` - (the same source template used by `mache deploy init/update`) + plus optional `deploy/custom_cli_spec.json` + - Uses the same source template as `mache deploy init/update` for the + generated portion **Benefits** - Single source of truth for user-facing CLI - No duplication across repositories - Consistent `--help` output +- Safe downstream extension point for repo-specific flags --- @@ -328,6 +333,7 @@ packaged template, which declaratively defines: - `deploy.py` - `cli_spec.json.j2` +- `custom_cli_spec.json` - `pins.cfg` - `config.yaml.j2.j2` (renders to `deploy/config.yaml.j2`) - `pixi.toml.j2.j2` (renders to `deploy/pixi.toml.j2`) @@ -352,8 +358,9 @@ A command that: - Fills in required placeholders - Creates a minimal, working deployment setup - Writes: `deploy.py`, `deploy/cli_spec.json`, `deploy/pins.cfg`, - `deploy/config.yaml.j2`, `deploy/pixi.toml.j2`, `deploy/spack.yaml.j2`, - `deploy/hooks.py`, and a placeholder `deploy/load.sh` + `deploy/custom_cli_spec.json`, `deploy/config.yaml.j2`, + `deploy/pixi.toml.j2`, `deploy/spack.yaml.j2`, `deploy/hooks.py`, and a + placeholder `deploy/load.sh` --- @@ -363,8 +370,8 @@ When a target software updates its pinned `mache` version: - `deploy.py` and `deploy/cli_spec.json` should be updated from the matching `mache` release -- `deploy/pins.cfg`, `deploy/config.yaml.j2`, and `deploy/pixi.toml.j2` remain - software-owned +- `deploy/custom_cli_spec.json`, `deploy/pins.cfg`, `deploy/config.yaml.j2`, + and `deploy/pixi.toml.j2` remain software-owned The `mache deploy update` command updates only `deploy.py` and `deploy/cli_spec.json`. diff --git a/docs/developers_guide/api.md b/docs/developers_guide/api.md index 88ca9e47..1ba7af83 100644 --- a/docs/developers_guide/api.md +++ b/docs/developers_guide/api.md @@ -175,9 +175,11 @@ documentation. CliArgSpec CliSpec parse_cli_spec + merge_cli_specs routes_include filter_args_by_route load_cli_spec_file + load_repo_cli_spec_file add_args_to_parser ``` diff --git a/docs/developers_guide/deploy.md b/docs/developers_guide/deploy.md index f7a62ffe..0d24f3a2 100644 --- a/docs/developers_guide/deploy.md +++ b/docs/developers_guide/deploy.md @@ -78,6 +78,10 @@ These are the only files currently refreshed by `mache deploy update`. ### Rendered only during `mache deploy init` +`custom_cli_spec.json` +: Rendered into `deploy/custom_cli_spec.json` as a downstream-owned extension + point for extra CLI flags. + `pins.cfg.j2` : Rendered into `deploy/pins.cfg`. @@ -144,6 +148,7 @@ Generated and expected to track `mache` closely: Target-owned after `init`: +- `deploy/custom_cli_spec.json` - `deploy/pins.cfg` - `deploy/config.yaml.j2` - `deploy/pixi.toml.j2` @@ -167,7 +172,8 @@ target ownership of `deploy/pins.cfg` is more important than auto-rewriting it. ## The command-line contract from the maintainer side The runtime CLI surface is defined by `mache/deploy/templates/cli_spec.json.j2` -and consumed in three places: +plus an optional downstream-owned `deploy/custom_cli_spec.json`, and consumed +in three places: 1. Target-side `deploy.py` reads the rendered JSON and exposes the user-facing arguments. @@ -184,9 +190,11 @@ When changing the CLI contract: 5. Update or add tests. One important subtlety is that `mache deploy run` builds its parser from the -package template, while target repositories use their rendered -`deploy/cli_spec.json`. A downstream repo therefore remains safe only when its -rendered JSON stays compatible with the pinned `mache` version. +package template plus optional `deploy/custom_cli_spec.json`, while target +repositories use their rendered `deploy/cli_spec.json` plus the same optional +custom file. A downstream repo therefore remains safe only when its rendered +JSON stays compatible with the pinned `mache` version and its custom entries +avoid duplicate `flags` and `dest` values. ## Changing starter-kit generation diff --git a/docs/users_guide/deploy.md b/docs/users_guide/deploy.md index e13a22ec..4cb6d5fe 100644 --- a/docs/users_guide/deploy.md +++ b/docs/users_guide/deploy.md @@ -43,6 +43,7 @@ deploy.py deploy/ cli_spec.json config.yaml.j2 + custom_cli_spec.json hooks.py load.sh pins.cfg @@ -119,6 +120,31 @@ Edit policy: - Any argument you add must remain compatible with the bootstrap parser and `mache deploy run` parser in the installed `mache` version. +### `deploy/custom_cli_spec.json` + +Required: no + +Created by `mache deploy init`: yes + +Overwritten by `mache deploy update`: no + +Purpose: + +- Lets the target repository add its own command-line flags without editing + the generated `deploy/cli_spec.json`. +- Is merged into the generated CLI by both `./deploy.py` and + `mache deploy run`. +- Is the recommended place for downstream-only CLI additions such as optional + feature toggles. + +Edit policy: + +- Fully target-repository owned. +- Safe to edit freely after `init`. +- Keep custom `flags` and `dest` names distinct from the generated + `deploy/cli_spec.json`. +- Omit `meta`; only `arguments` is needed. + ### `deploy/pins.cfg` Required: yes @@ -280,6 +306,85 @@ them: target repository needs a custom environment skeleton for one machine or toolchain combination. +## Adding a custom CLI flag + +Use `deploy/custom_cli_spec.json` when the target repository wants an extra +flag that should survive `mache deploy update`. + +For example, Compass could add both `--with-albany` and +`--moab-version 5.6.0` like this: + +```json +{ + "arguments": [ + { + "flags": ["--with-albany"], + "dest": "with_albany", + "action": "store_true", + "help": "Install optional Albany and Trilinos support.", + "route": ["deploy", "run"] + }, + { + "flags": ["--moab-version"], + "dest": "moab_version", + "help": "Version of MOAB to install for this deployment.", + "route": ["deploy", "run"] + } + ] +} +``` + +This route means: + +- `deploy`: `./deploy.py` accepts the flag. +- `run`: the flag is forwarded to `mache deploy run`, so hooks can inspect it. + +Then enable hooks in `deploy/config.yaml.j2`: + +```yaml +hooks: + file: "deploy/hooks.py" + entrypoints: + pre_spack: "pre_spack" +``` + +And use the flags inside `deploy/hooks.py`: + +```python +from pathlib import Path + + +def pre_spack(ctx): + if getattr(ctx.args, "with_albany", False): + marker = Path(ctx.work_dir) / "with_albany.txt" + marker.write_text("Albany requested\n", encoding="utf-8") + + moab_version = getattr(ctx.args, "moab_version", None) + if moab_version: + marker = Path(ctx.work_dir) / "moab_version.txt" + marker.write_text(f"{moab_version}\n", encoding="utf-8") +``` + +Running + +```bash +./deploy.py --with-albany --moab-version 5.6.0 +``` + +would therefore make both of these values available in hooks: + +- `ctx.args.with_albany == True` +- `ctx.args.moab_version == "5.6.0"` + +From there, the hook can: + +- add derived runtime settings, +- render target-specific files, +- or trigger downstream install steps for optional dependencies. + +Use `deploy/custom_cli_spec.json` for downstream extensions. Keep +`deploy/cli_spec.json` aligned with upstream `mache`. + ## The three deployment phases From a user's point of view, deployment starts with `./deploy.py`. Internally, @@ -291,7 +396,8 @@ The generated `deploy.py` file: 1. Verifies it is being run from the target repository root. 2. Reads the pinned `mache` and Python versions from `deploy/pins.cfg`. -3. Reads `deploy/cli_spec.json` and builds the command-line parser. +3. Reads `deploy/cli_spec.json` plus optional `deploy/custom_cli_spec.json` + and builds the command-line parser. 4. Validates that `deploy/cli_spec.json` and `deploy/pins.cfg` agree on the pinned `mache` version unless a fork/branch override is being used. 5. Downloads `mache/deploy/bootstrap.py` from the requested `mache` source. @@ -383,6 +489,9 @@ These commands operate on the target repository itself. : Regenerates only the files that are intended to track `mache` closely, currently `deploy.py` and `deploy/cli_spec.json`. +`deploy/custom_cli_spec.json` +: Is intentionally not regenerated. Use it for target-owned CLI additions. + ### How `deploy/cli_spec.json` works Each argument entry contains: @@ -414,15 +523,34 @@ Examples from the current contract: - `--pixi`, `--prefix`, `--recreate`, `--quiet`, `--mache-fork`, and `--mache-branch` are routed across all relevant phases. +### How `deploy/custom_cli_spec.json` works + +`deploy/custom_cli_spec.json` uses the same argument-entry format as +`deploy/cli_spec.json`, but it is downstream-owned and optional. + +At runtime: + +1. `./deploy.py` loads `deploy/cli_spec.json`. +2. If `deploy/custom_cli_spec.json` exists, it appends those argument entries. +3. Duplicate `flags` or `dest` values are rejected to avoid ambiguous parsers. + +In practice, this means: + +- use `deploy/cli_spec.json` for the generated upstream contract, +- use `deploy/custom_cli_spec.json` for downstream feature flags, +- route custom flags to `run` when hooks need to inspect them. + ### Contract rules for target-software developers -When you customize `deploy/cli_spec.json`, keep these rules in mind: +When you customize `deploy/cli_spec.json` or `deploy/custom_cli_spec.json`, +keep these rules in mind: 1. `deploy.py` exposes only the arguments listed in the JSON file. 2. Bootstrap must accept every argument routed to `bootstrap`. 3. `mache deploy run` must accept every argument routed to `run`. 4. The pinned `mache` version in `deploy/pins.cfg` must match `deploy/cli_spec.json` unless you are intentionally testing a fork/branch. +5. `deploy/custom_cli_spec.json` must not reuse a generated flag or `dest`. If you break this contract, deployment usually fails with an argument-parsing error or a version-mismatch error. @@ -451,14 +579,16 @@ The usual sequence is: 3. Update `deploy/pins.cfg`, especially `[pixi] mache = 2.2.0`. 4. Exit the bootstrap shell. 5. Review the diffs in `deploy.py` and `deploy/cli_spec.json`. -6. Adjust repository-owned files such as `deploy/config.yaml.j2` only if the +6. Confirm that target-owned files such as `deploy/custom_cli_spec.json` still + reflect the downstream CLI you want. +7. Adjust repository-owned files such as `deploy/config.yaml.j2` only if the new `mache` release expects new settings or supports new behavior. Important limitation: - `mache deploy update` does not rewrite `deploy/pins.cfg`, - `deploy/config.yaml.j2`, `deploy/pixi.toml.j2`, `deploy/spack.yaml.j2`, or - `deploy/hooks.py`. + `deploy/config.yaml.j2`, `deploy/custom_cli_spec.json`, + `deploy/pixi.toml.j2`, `deploy/spack.yaml.j2`, or `deploy/hooks.py`. - `./deploy.py --bootstrap-only --mache-version ` is the safest way to make sure the bootstrap environment and downloaded `bootstrap.py` come from the new release instead of the old pin. diff --git a/mache/deploy/templates/config.yaml.j2.j2 b/mache/deploy/templates/config.yaml.j2.j2 index 2ac166bf..a42c5d8b 100644 --- a/mache/deploy/templates/config.yaml.j2.j2 +++ b/mache/deploy/templates/config.yaml.j2.j2 @@ -189,6 +189,8 @@ jigsaw: # # Hooks run ONLY during `mache deploy run` and execute arbitrary Python code # from the target repository. Use only with trusted repositories. +# Custom CLI flags from deploy/custom_cli_spec.json are available on ctx.args +# inside hooks when routed to "run". # # hooks: # file: "deploy/hooks.py" # default diff --git a/mache/deploy/templates/hooks.py.j2 b/mache/deploy/templates/hooks.py.j2 index b4c2f4f9..2d705aaa 100644 --- a/mache/deploy/templates/hooks.py.j2 +++ b/mache/deploy/templates/hooks.py.j2 @@ -13,6 +13,29 @@ To enable, add a `hooks` section like: post_pixi: "post_pixi" # optional post_deploy: "post_deploy" # optional +Custom CLI flags added in `deploy/custom_cli_spec.json` are available on +`ctx.args` when their route includes `"run"`. + +Example `deploy/custom_cli_spec.json` entry: + + { + "arguments": [ + { + "flags": ["--with-albany"], + "dest": "with_albany", + "action": "store_true", + "help": "Install optional Albany and Trilinos support.", + "route": ["deploy", "run"] + }, + { + "flags": ["--moab-version"], + "dest": "moab_version", + "help": "Version of MOAB to install for this deployment.", + "route": ["deploy", "run"] + } + ] + } + """ from __future__ import annotations @@ -59,6 +82,13 @@ def pre_pixi(ctx: DeployContext) -> dict[str, Any] | None: def post_pixi(ctx: DeployContext) -> None: """Run after the pixi environment is created/updated.""" + # Example: downstream-owned CLI flags are available on ctx.args + if getattr(ctx.args, "with_albany", False): + ctx.logger.info("Optional Albany support was requested") + moab_version = getattr(ctx.args, "moab_version", None) + if moab_version: + ctx.logger.info("Using MOAB version override: %s", moab_version) + # Example: write a small marker into deploy_tmp marker = os.path.join(ctx.work_dir, "hooks_post_pixi_ran.txt") os.makedirs(ctx.work_dir, exist_ok=True) diff --git a/workflow_tests/fixtures/deploy_hook_overrides/custom_cli_spec.json b/workflow_tests/fixtures/deploy_hook_overrides/custom_cli_spec.json index a4999c70..2ef4a285 100644 --- a/workflow_tests/fixtures/deploy_hook_overrides/custom_cli_spec.json +++ b/workflow_tests/fixtures/deploy_hook_overrides/custom_cli_spec.json @@ -6,6 +6,12 @@ "action": "store_true", "help": "Install optional Albany support.", "route": ["deploy", "run"] + }, + { + "flags": ["--moab-version"], + "dest": "moab_version", + "help": "Version of MOAB to install for this deployment.", + "route": ["deploy", "run"] } ] } diff --git a/workflow_tests/fixtures/deploy_hook_overrides/hooks.py b/workflow_tests/fixtures/deploy_hook_overrides/hooks.py index aa3250df..d3d85ff9 100644 --- a/workflow_tests/fixtures/deploy_hook_overrides/hooks.py +++ b/workflow_tests/fixtures/deploy_hook_overrides/hooks.py @@ -5,3 +5,8 @@ def post_pixi(ctx): if getattr(ctx.args, 'with_albany', False): marker = Path(ctx.work_dir) / 'with_albany.txt' marker.write_text('enabled\n', encoding='utf-8') + + moab_version = getattr(ctx.args, 'moab_version', None) + if moab_version: + marker = Path(ctx.work_dir) / 'moab_version.txt' + marker.write_text(f'{moab_version}\n', encoding='utf-8') diff --git a/workflow_tests/test_deploy_workflow.py b/workflow_tests/test_deploy_workflow.py index 3eed2f06..1d87d4d2 100644 --- a/workflow_tests/test_deploy_workflow.py +++ b/workflow_tests/test_deploy_workflow.py @@ -71,6 +71,8 @@ def test_downstream_deploy_workflow(tmp_path: Path): '--mache-branch', 'current', '--with-albany', + '--moab-version', + '5.6.0', '--recreate', ], cwd=downstream, @@ -82,6 +84,9 @@ def test_downstream_deploy_workflow(tmp_path: Path): assert (downstream / 'deploy_tmp' / 'with_albany.txt').read_text( encoding='utf-8' ) == 'enabled\n' + assert (downstream / 'deploy_tmp' / 'moab_version.txt').read_text( + encoding='utf-8' + ) == '5.6.0\n' smoke = run( [