From d51d17b47da55a9ef5c4b179ed3f9668576b5f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 3 Apr 2025 16:52:14 +0300 Subject: [PATCH 01/30] Sketch multi-version scripts. --- .flake8 | 2 + multiversion/README.md | 25 ++++++ multiversion/build.json | 22 +++++ multiversion/build.py | 184 ++++++++++++++++++++++++++++++++++++++ multiversion/constants.py | 6 ++ multiversion/driver.py | 28 ++++++ multiversion/errors.py | 29 ++++++ multiversion/platform.py | 27 ++++++ pyrightconfig.json | 14 +++ requirements.txt | 2 + 10 files changed, 339 insertions(+) create mode 100644 .flake8 create mode 100644 multiversion/README.md create mode 100644 multiversion/build.json create mode 100644 multiversion/build.py create mode 100644 multiversion/constants.py create mode 100644 multiversion/driver.py create mode 100644 multiversion/errors.py create mode 100644 multiversion/platform.py create mode 100644 pyrightconfig.json create mode 100644 requirements.txt diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..e44b810 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +ignore = E501 diff --git a/multiversion/README.md b/multiversion/README.md new file mode 100644 index 0000000..a30ac90 --- /dev/null +++ b/multiversion/README.md @@ -0,0 +1,25 @@ +# MultiversX multi-version node scripts + +These scripts allow one to use multiple versions of the MultiversX node, in sequence, to _sync_ (from the deep past) or run _import-db_ flows. + +**Important:** these scripts are only suitable for observers, not for validators. Furthermore, the MultiversX proxy isn't handled. + +## Building the binaries + +Go must be installed beforehand. + +``` +PYTHONPATH=. python3 ./multiversion/build.py --workspace=~/mvx-workspace --config=./multiversion/build.json +``` + +## Maintenance + +### Python virtual environment + +Create a virtual environment and install the dependencies: + +``` +python3 -m venv ./venv +source ./venv/bin/activate +pip install -r ./requirements.txt --upgrade +``` diff --git a/multiversion/build.json b/multiversion/build.json new file mode 100644 index 0000000..806f9c4 --- /dev/null +++ b/multiversion/build.json @@ -0,0 +1,22 @@ +[ + { + "name": "sirius", + "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.6.18.zip", + "destinationFolder": "~/binaries/sirius" + }, + { + "name": "vega", + "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.7.13-patch2.zip", + "destinationFolder": "~/binaries/vega" + }, + { + "name": "spica", + "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.8.12.zip", + "destinationFolder": "~/binaries/spica" + }, + { + "name": "andromeda", + "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.9.0.zip", + "destinationFolder": "~/binaries/andromeda" + } +] diff --git a/multiversion/build.py b/multiversion/build.py new file mode 100644 index 0000000..284e196 --- /dev/null +++ b/multiversion/build.py @@ -0,0 +1,184 @@ +import json +import os +import shutil +import subprocess +import sys +import traceback +import urllib.request +from argparse import ArgumentParser +from pathlib import Path +from typing import Any, List + +from rich import print +from rich.panel import Panel +from rich.rule import Rule + +from multiversion import errors, platform +from multiversion.constants import FILE_MODE_EXECUTABLE + + +class BuildConfigEntry: + def __init__(self, name: str, source_url: str, destination_folder: str) -> None: + if not name: + raise errors.KnownError("build 'name' is required") + if not source_url: + raise errors.KnownError("build 'source' is required") + if not destination_folder: + raise errors.KnownError("build 'destination' is required") + + self.name = name + self.source_url = source_url + self.destination_folder = destination_folder + + @classmethod + def new_from_dictionary(cls, data: dict[str, Any]): + name = data.get("name") or "" + source_url = data.get("sourceUrl") or "" + destination_folder = data.get("destinationFolder") or "" + + return cls( + name=name, + source_url=source_url, + destination_folder=destination_folder, + ) + + +def main(cli_args: list[str] = sys.argv[1:]): + try: + _do_main(cli_args) + except errors.KnownError as err: + print(Panel(f"[red]{traceback.format_exc()}")) + print(Panel(f"[red]{err.get_pretty()}")) + return 1 + + +def _do_main(cli_args: list[str]): + parser = ArgumentParser() + parser.add_argument("--workspace", required=True, help="path of the build workspace") + parser.add_argument("--config", required=True, help="path of the 'build' configuration file") + args = parser.parse_args(cli_args) + + workspace_path = Path(args.workspace).expanduser().resolve() + workspace_path.mkdir(parents=True, exist_ok=True) + + config_path = Path(args.config).expanduser().resolve() + config_data = json.loads(config_path.read_text()) + config_entries = [BuildConfigEntry.new_from_dictionary(item) for item in config_data] + + for entry in config_entries: + print(Rule(f"[bold yellow]{entry.name}")) + do_download(workspace_path, entry) + do_build(workspace_path, entry) + + +def do_download(workspace: Path, entry: BuildConfigEntry): + download_folder = workspace / "downloads" / entry.name + extraction_folder = get_extraction_folder(workspace, entry) + url = entry.source_url + + print(f"Re-creating {download_folder} ...") + shutil.rmtree(download_folder, ignore_errors=True) + download_folder.mkdir(parents=True, exist_ok=True) + + print(f"Re-creating {extraction_folder} ...") + shutil.rmtree(extraction_folder, ignore_errors=True) + extraction_folder.mkdir(parents=True, exist_ok=True) + + archive_extension = url.split(".")[-1] + download_path = download_folder / f"source.{archive_extension}" + + print(f"Downloading archive {url} to {download_path}") + urllib.request.urlretrieve(url, download_path) + + print(f"Unpacking archive {download_path} to {extraction_folder}") + shutil.unpack_archive(download_path, extraction_folder, format="zip") + + +def get_extraction_folder(workspace: Path, entry: BuildConfigEntry): + return workspace / "builds" / entry.name + + +def do_build(workspace: Path, entry: BuildConfigEntry): + extraction_folder = get_extraction_folder(workspace, entry) + source_folder = locate_source_folder_in_archive_extraction_folder(extraction_folder) + cmd_node = source_folder / "cmd" / "node" + go_mod = source_folder / "go.mod" + + print(f"Building {cmd_node} ...") + + return_code = subprocess.check_call(["go", "build"], cwd=cmd_node) + if return_code != 0: + raise errors.KnownError(f"error code = {return_code}, see output") + + copy_wasmer_libs(go_mod, cmd_node) + set_rpath(cmd_node / "node") + + +def locate_source_folder_in_archive_extraction_folder(extraction_folder: Path) -> Path: + # If has one subfolder, that one is the source code + subfolders = list(extraction_folder.glob("*")) + source_folder = subfolders[0] if len(subfolders) == 1 else extraction_folder + + # Heuristic to check if this is a valid source code folder + assert (source_folder / "go.mod").exists(), f"This is not a valid source code folder: {source_folder}" + return source_folder + + +def copy_wasmer_libs(go_mod: Path, destination: Path): + go_path_variable = os.environ.get("GOPATH", "~/go") + go_path = Path(go_path_variable).expanduser().resolve() + vm_go_folder_name = get_chain_vm_go_folder_name(go_mod) + vm_go_path = go_path / "pkg" / "mod" / vm_go_folder_name + wasmer_path = vm_go_path / "wasmer" + wasmer2_path = vm_go_path / "wasmer2" + + copy_libraries(wasmer_path, destination) + copy_libraries(wasmer2_path, destination) + + +def get_chain_vm_go_folder_name(go_mod: Path) -> str: + lines = go_mod.read_text().splitlines() + line = [line for line in lines if "github.com/multiversx/mx-chain-vm-go" in line][0] + parts = line.split() + return f"{parts[0]}@{parts[1]}" + + +def copy_libraries(source: Path, destination: Path): + libraries: List[Path] = list(source.glob("*.dylib")) + list(source.glob("*.so")) + + for library in libraries: + print(f"Copying {library} to {destination}") + shutil.copy(library, destination) + + # Seems to be necessary on MacOS (or, at least, was necessary in the past). + os.chmod(destination / library.name, FILE_MODE_EXECUTABLE) + + +def set_rpath(cmd_path: Path): + """ + Set the rpath of the executable to the current directory, on a best-effort basis. + + For other occurrences of this approach, see: + - https://github.com/multiversx/mx-chain-scenario-cli-go/blob/master/.github/workflows/on_release_attach_artifacts.yml + """ + + if not platform.is_osx(): + # We're only patching the executable on macOS. + # For Linux, we're leveraging LD_LIBRARY_PATH to resolve the libraries. + return + + try: + subprocess.check_call([ + "install_name_tool", + "-add_rpath", + "@loader_path", + cmd_path + ]) + except Exception as e: + # In most cases, this isn't critical (libraries might be found among the downloaded Go packages). + print(f"Failed to set rpath of {cmd_path}: {e}") + + +if __name__ == "__main__": + ret = main(sys.argv[1:]) + sys.exit(ret) diff --git a/multiversion/constants.py b/multiversion/constants.py new file mode 100644 index 0000000..9800f33 --- /dev/null +++ b/multiversion/constants.py @@ -0,0 +1,6 @@ +import stat + +METACHAIN_ID = 4294967295 + +# Read, write and execute by owner, read and execute by group and others +FILE_MODE_EXECUTABLE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH diff --git a/multiversion/driver.py b/multiversion/driver.py new file mode 100644 index 0000000..f16b99c --- /dev/null +++ b/multiversion/driver.py @@ -0,0 +1,28 @@ +import sys +import traceback +from argparse import ArgumentParser + +from rich import print +from rich.panel import Panel + +from multiversion import errors + + +def main(cli_args: list[str] = sys.argv[1:]): + try: + _do_main(cli_args) + except errors.KnownError as err: + print(Panel(f"[red]{traceback.format_exc()}")) + print(Panel(f"[red]{err.get_pretty()}")) + return 1 + + +def _do_main(cli_args: list[str]): + parser = ArgumentParser() + args = parser.parse_args(cli_args) + + +if __name__ == "__main__": + ret = main(sys.argv[1:]) + sys.exit(ret) + diff --git a/multiversion/errors.py b/multiversion/errors.py new file mode 100644 index 0000000..5b84e62 --- /dev/null +++ b/multiversion/errors.py @@ -0,0 +1,29 @@ +from typing import Any, Optional + + +class KnownError(Exception): + def __init__(self, message: str, inner: Optional[Any] = None): + super().__init__(message) + self.inner = inner + + def get_pretty(self) -> str: + if self.inner: + return f"""{self} +... {self.inner} +""" + return str(self) + + +class BadConfigurationError(KnownError): + def __init__(self, message: str): + super().__init__(f"bad configuration: {message}") + + +class UsageError(KnownError): + def __init__(self, message: str): + super().__init__(f"bad usage: {message}") + + +class TransientError(KnownError): + def __init__(self, message: str, inner: Optional[Any] = None): + super().__init__(f"transient error: {message}", inner) diff --git a/multiversion/platform.py b/multiversion/platform.py new file mode 100644 index 0000000..248bd09 --- /dev/null +++ b/multiversion/platform.py @@ -0,0 +1,27 @@ +import sys + + +def is_linux(): + return get_platform() == "linux" + + +def is_osx(): + return get_platform() == "osx" + + +def get_platform(): + platforms = { + "linux": "linux", + "linux1": "linux", + "linux2": "linux", + "darwin": "osx", + "win32": "windows", + "cygwin": "windows", + "msys": "windows" + } + + platform = platforms.get(sys.platform) + if platform is None: + raise Exception(f"Unknown platform: {sys.platform}") + + return platform diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..72a3c50 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,14 @@ +{ + "include": ["multiversion"], + "exclude": ["**/__pycache__"], + "ignore": [], + "defineConstant": { + "DEBUG": true + }, + "venvPath": ".", + "venv": "venv", + "stubPath": "", + "reportMissingImports": true, + "reportMissingTypeStubs": false, + "reportUnknownParameterType": true +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9b0bc41 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests>=2.32.0,<3.0.0 +rich==13.9.4 From 1320df3399fab5ac744af149e643e0511bab9b20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 3 Apr 2025 16:53:33 +0300 Subject: [PATCH 02/30] Actually, add only Linux support. --- .gitignore | 2 ++ multiversion/build.py | 34 ++-------------------------------- multiversion/constants.py | 3 --- multiversion/platform.py | 27 --------------------------- 4 files changed, 4 insertions(+), 62 deletions(-) delete mode 100644 multiversion/platform.py diff --git a/.gitignore b/.gitignore index 1563977..0eaa8a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ #Don't track files: .DS_Store +venv/ +__pycache__ diff --git a/multiversion/build.py b/multiversion/build.py index 284e196..d8addb1 100644 --- a/multiversion/build.py +++ b/multiversion/build.py @@ -13,8 +13,7 @@ from rich.panel import Panel from rich.rule import Rule -from multiversion import errors, platform -from multiversion.constants import FILE_MODE_EXECUTABLE +from multiversion import errors class BuildConfigEntry: @@ -111,7 +110,6 @@ def do_build(workspace: Path, entry: BuildConfigEntry): raise errors.KnownError(f"error code = {return_code}, see output") copy_wasmer_libs(go_mod, cmd_node) - set_rpath(cmd_node / "node") def locate_source_folder_in_archive_extraction_folder(extraction_folder: Path) -> Path: @@ -144,40 +142,12 @@ def get_chain_vm_go_folder_name(go_mod: Path) -> str: def copy_libraries(source: Path, destination: Path): - libraries: List[Path] = list(source.glob("*.dylib")) + list(source.glob("*.so")) + libraries: List[Path] = list(source.glob("*.so")) for library in libraries: print(f"Copying {library} to {destination}") shutil.copy(library, destination) - # Seems to be necessary on MacOS (or, at least, was necessary in the past). - os.chmod(destination / library.name, FILE_MODE_EXECUTABLE) - - -def set_rpath(cmd_path: Path): - """ - Set the rpath of the executable to the current directory, on a best-effort basis. - - For other occurrences of this approach, see: - - https://github.com/multiversx/mx-chain-scenario-cli-go/blob/master/.github/workflows/on_release_attach_artifacts.yml - """ - - if not platform.is_osx(): - # We're only patching the executable on macOS. - # For Linux, we're leveraging LD_LIBRARY_PATH to resolve the libraries. - return - - try: - subprocess.check_call([ - "install_name_tool", - "-add_rpath", - "@loader_path", - cmd_path - ]) - except Exception as e: - # In most cases, this isn't critical (libraries might be found among the downloaded Go packages). - print(f"Failed to set rpath of {cmd_path}: {e}") - if __name__ == "__main__": ret = main(sys.argv[1:]) diff --git a/multiversion/constants.py b/multiversion/constants.py index 9800f33..b6e8de9 100644 --- a/multiversion/constants.py +++ b/multiversion/constants.py @@ -1,6 +1,3 @@ import stat METACHAIN_ID = 4294967295 - -# Read, write and execute by owner, read and execute by group and others -FILE_MODE_EXECUTABLE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH diff --git a/multiversion/platform.py b/multiversion/platform.py deleted file mode 100644 index 248bd09..0000000 --- a/multiversion/platform.py +++ /dev/null @@ -1,27 +0,0 @@ -import sys - - -def is_linux(): - return get_platform() == "linux" - - -def is_osx(): - return get_platform() == "osx" - - -def get_platform(): - platforms = { - "linux": "linux", - "linux1": "linux", - "linux2": "linux", - "darwin": "osx", - "win32": "windows", - "cygwin": "windows", - "msys": "windows" - } - - platform = platforms.get(sys.platform) - if platform is None: - raise Exception(f"Unknown platform: {sys.platform}") - - return platform From 1b3e7e082e959e995ee2646e073277b84d94bd1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 3 Apr 2025 17:46:25 +0300 Subject: [PATCH 03/30] Adjust / finalize build flow. --- multiversion/build.json | 8 ++--- multiversion/build.py | 64 +++++++++++++++++++-------------------- multiversion/constants.py | 3 ++ 3 files changed, 39 insertions(+), 36 deletions(-) diff --git a/multiversion/build.json b/multiversion/build.json index 806f9c4..fa521b2 100644 --- a/multiversion/build.json +++ b/multiversion/build.json @@ -2,21 +2,21 @@ { "name": "sirius", "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.6.18.zip", - "destinationFolder": "~/binaries/sirius" + "destinationFolder": "~/mvx-binaries/sirius" }, { "name": "vega", "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.7.13-patch2.zip", - "destinationFolder": "~/binaries/vega" + "destinationFolder": "~/mvx-binaries/vega" }, { "name": "spica", "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.8.12.zip", - "destinationFolder": "~/binaries/spica" + "destinationFolder": "~/mvx-binaries/spica" }, { "name": "andromeda", "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.9.0.zip", - "destinationFolder": "~/binaries/andromeda" + "destinationFolder": "~/mvx-binaries/andromeda" } ] diff --git a/multiversion/build.py b/multiversion/build.py index d8addb1..5a01b43 100644 --- a/multiversion/build.py +++ b/multiversion/build.py @@ -7,13 +7,14 @@ import urllib.request from argparse import ArgumentParser from pathlib import Path -from typing import Any, List +from typing import Any from rich import print from rich.panel import Panel from rich.rule import Rule from multiversion import errors +from multiversion.constants import FILE_MODE_NICE class BuildConfigEntry: @@ -66,13 +67,14 @@ def _do_main(cli_args: list[str]): for entry in config_entries: print(Rule(f"[bold yellow]{entry.name}")) - do_download(workspace_path, entry) - do_build(workspace_path, entry) + source_parent_folder = do_download(workspace_path, entry) + cmd_node_folder = do_build(source_parent_folder) + copy_artifacts(cmd_node_folder, entry) -def do_download(workspace: Path, entry: BuildConfigEntry): - download_folder = workspace / "downloads" / entry.name - extraction_folder = get_extraction_folder(workspace, entry) +def do_download(workspace: Path, entry: BuildConfigEntry) -> Path: + download_folder = workspace / entry.name + extraction_folder = workspace / entry.name url = entry.source_url print(f"Re-creating {download_folder} ...") @@ -92,14 +94,14 @@ def do_download(workspace: Path, entry: BuildConfigEntry): print(f"Unpacking archive {download_path} to {extraction_folder}") shutil.unpack_archive(download_path, extraction_folder, format="zip") + return extraction_folder -def get_extraction_folder(workspace: Path, entry: BuildConfigEntry): - return workspace / "builds" / entry.name +def do_build(source_parent_folder: Path) -> Path: + # If has one subfolder, that one is the source code + subfolders = [Path(item.path) for item in os.scandir(source_parent_folder) if item.is_dir()] + source_folder = subfolders[0] if len(subfolders) == 1 else source_parent_folder -def do_build(workspace: Path, entry: BuildConfigEntry): - extraction_folder = get_extraction_folder(workspace, entry) - source_folder = locate_source_folder_in_archive_extraction_folder(extraction_folder) cmd_node = source_folder / "cmd" / "node" go_mod = source_folder / "go.mod" @@ -109,29 +111,20 @@ def do_build(workspace: Path, entry: BuildConfigEntry): if return_code != 0: raise errors.KnownError(f"error code = {return_code}, see output") - copy_wasmer_libs(go_mod, cmd_node) - - -def locate_source_folder_in_archive_extraction_folder(extraction_folder: Path) -> Path: - # If has one subfolder, that one is the source code - subfolders = list(extraction_folder.glob("*")) - source_folder = subfolders[0] if len(subfolders) == 1 else extraction_folder - - # Heuristic to check if this is a valid source code folder - assert (source_folder / "go.mod").exists(), f"This is not a valid source code folder: {source_folder}" - return source_folder + copy_wasmer_libraries(go_mod, cmd_node) + return cmd_node -def copy_wasmer_libs(go_mod: Path, destination: Path): +def copy_wasmer_libraries(go_mod: Path, destination: Path): go_path_variable = os.environ.get("GOPATH", "~/go") go_path = Path(go_path_variable).expanduser().resolve() vm_go_folder_name = get_chain_vm_go_folder_name(go_mod) vm_go_path = go_path / "pkg" / "mod" / vm_go_folder_name - wasmer_path = vm_go_path / "wasmer" - wasmer2_path = vm_go_path / "wasmer2" + libraries = list((vm_go_path / "wasmer").glob("*.so")) + list((vm_go_path / "wasmer2").glob("*.so")) - copy_libraries(wasmer_path, destination) - copy_libraries(wasmer2_path, destination) + for library in libraries: + shutil.copy(library, destination) + os.chmod(destination / library.name, FILE_MODE_NICE) def get_chain_vm_go_folder_name(go_mod: Path) -> str: @@ -141,12 +134,19 @@ def get_chain_vm_go_folder_name(go_mod: Path) -> str: return f"{parts[0]}@{parts[1]}" -def copy_libraries(source: Path, destination: Path): - libraries: List[Path] = list(source.glob("*.so")) +def copy_artifacts(cmd_node_folder: Path, entry: BuildConfigEntry): + print(f"Copying artifacts to {entry.destination_folder} ...") - for library in libraries: - print(f"Copying {library} to {destination}") - shutil.copy(library, destination) + libraries = list(cmd_node_folder.glob("*.so")) + executable = cmd_node_folder / "node" + artifacts = libraries + [executable] + + destination_folder = Path(entry.destination_folder).expanduser().resolve() + shutil.rmtree(destination_folder, ignore_errors=True) + destination_folder.mkdir(parents=True, exist_ok=True) + + for artifact in artifacts: + shutil.copy(artifact, destination_folder) if __name__ == "__main__": diff --git a/multiversion/constants.py b/multiversion/constants.py index b6e8de9..a791d12 100644 --- a/multiversion/constants.py +++ b/multiversion/constants.py @@ -1,3 +1,6 @@ import stat METACHAIN_ID = 4294967295 + +# Read, write and execute by owner, read and execute by group and others +FILE_MODE_NICE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH From 2b60f56f744045fa4dd762fdff82c4a350141e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 3 Apr 2025 17:47:48 +0300 Subject: [PATCH 04/30] Update readme. --- multiversion/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/multiversion/README.md b/multiversion/README.md index a30ac90..3d3d6f7 100644 --- a/multiversion/README.md +++ b/multiversion/README.md @@ -4,7 +4,9 @@ These scripts allow one to use multiple versions of the MultiversX node, in sequ **Important:** these scripts are only suitable for observers, not for validators. Furthermore, the MultiversX proxy isn't handled. -## Building the binaries +## Building the artifacts + +Skip this flow if you choose to download the pre-built Node artifacts, instead of building them. Go must be installed beforehand. From c9a2a5bbcdd542f909baef0063585d1a2ddb209a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 30 Jun 2025 15:29:24 +0300 Subject: [PATCH 05/30] Minor update - readme, config. --- multiversion/README.md | 20 +++++++++----------- multiversion/build.json | 12 +++++++++++- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/multiversion/README.md b/multiversion/README.md index 3d3d6f7..28063bc 100644 --- a/multiversion/README.md +++ b/multiversion/README.md @@ -4,24 +4,22 @@ These scripts allow one to use multiple versions of the MultiversX node, in sequ **Important:** these scripts are only suitable for observers, not for validators. Furthermore, the MultiversX proxy isn't handled. -## Building the artifacts - -Skip this flow if you choose to download the pre-built Node artifacts, instead of building them. +## Python virtual environment -Go must be installed beforehand. +Create a virtual environment and install the dependencies: ``` -PYTHONPATH=. python3 ./multiversion/build.py --workspace=~/mvx-workspace --config=./multiversion/build.json +python3 -m venv ./venv +source ./venv/bin/activate +pip install -r ./requirements.txt --upgrade ``` -## Maintenance +## Building the artifacts -### Python virtual environment +Skip this flow if you choose to download the pre-built Node artifacts, instead of building them. -Create a virtual environment and install the dependencies: +Go must be installed beforehand. ``` -python3 -m venv ./venv -source ./venv/bin/activate -pip install -r ./requirements.txt --upgrade +PYTHONPATH=. python3 ./multiversion/build.py --workspace=~/mvx-workspace --config=./multiversion/build.json ``` diff --git a/multiversion/build.json b/multiversion/build.json index fa521b2..b6a2987 100644 --- a/multiversion/build.json +++ b/multiversion/build.json @@ -2,21 +2,31 @@ { "name": "sirius", "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.6.18.zip", + "goTag": "go1.20.7", "destinationFolder": "~/mvx-binaries/sirius" }, { "name": "vega", "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.7.13-patch2.zip", + "goTag": "go1.20.7", "destinationFolder": "~/mvx-binaries/vega" }, { "name": "spica", "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.8.12.zip", + "goTag": "go1.20.7", "destinationFolder": "~/mvx-binaries/spica" }, { "name": "andromeda", - "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.9.0.zip", + "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.9.6.zip", + "goTag": "go1.20.7", "destinationFolder": "~/mvx-binaries/andromeda" + }, + { + "name": "barnard", + "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.10.0.zip", + "goTag": "go1.23.10", + "destinationFolder": "~/mvx-binaries/barnard" } ] From ff4caf1a97f6ae7afe78e1f1e0042488ca15d544 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 1 Jul 2025 13:40:24 +0300 Subject: [PATCH 06/30] Build wrt. desired Go version. --- multiversion/build.json | 10 +++--- multiversion/build.py | 33 ++++++++++--------- multiversion/golang.py | 72 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 21 deletions(-) create mode 100644 multiversion/golang.py diff --git a/multiversion/build.json b/multiversion/build.json index b6a2987..1515332 100644 --- a/multiversion/build.json +++ b/multiversion/build.json @@ -1,32 +1,32 @@ [ { "name": "sirius", + "goUrl": "https://golang.org/dl/go1.20.7.linux-amd64.tar.gz", "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.6.18.zip", - "goTag": "go1.20.7", "destinationFolder": "~/mvx-binaries/sirius" }, { "name": "vega", + "goUrl": "https://golang.org/dl/go1.20.7.linux-amd64.tar.gz", "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.7.13-patch2.zip", - "goTag": "go1.20.7", "destinationFolder": "~/mvx-binaries/vega" }, { "name": "spica", + "goUrl": "https://golang.org/dl/go1.20.7.linux-amd64.tar.gz", "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.8.12.zip", - "goTag": "go1.20.7", "destinationFolder": "~/mvx-binaries/spica" }, { "name": "andromeda", + "goUrl": "https://golang.org/dl/go1.20.7.linux-amd64.tar.gz", "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.9.6.zip", - "goTag": "go1.20.7", "destinationFolder": "~/mvx-binaries/andromeda" }, { "name": "barnard", + "goUrl": "https://golang.org/dl/go1.23.10.linux-amd64.tar.gz", "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.10.0.zip", - "goTag": "go1.23.10", "destinationFolder": "~/mvx-binaries/barnard" } ] diff --git a/multiversion/build.py b/multiversion/build.py index 5a01b43..89dc76a 100644 --- a/multiversion/build.py +++ b/multiversion/build.py @@ -1,7 +1,6 @@ import json import os import shutil -import subprocess import sys import traceback import urllib.request @@ -13,31 +12,36 @@ from rich.panel import Panel from rich.rule import Rule -from multiversion import errors +from multiversion import errors, golang from multiversion.constants import FILE_MODE_NICE class BuildConfigEntry: - def __init__(self, name: str, source_url: str, destination_folder: str) -> None: + def __init__(self, name: str, go_url: str, source_url: str, destination_folder: str) -> None: if not name: raise errors.KnownError("build 'name' is required") + if not go_url: + raise errors.KnownError("build 'go url' is required") if not source_url: raise errors.KnownError("build 'source' is required") if not destination_folder: raise errors.KnownError("build 'destination' is required") self.name = name + self.go_url = go_url self.source_url = source_url self.destination_folder = destination_folder @classmethod def new_from_dictionary(cls, data: dict[str, Any]): name = data.get("name") or "" + go_url = data.get("goUrl") or "" source_url = data.get("sourceUrl") or "" destination_folder = data.get("destinationFolder") or "" return cls( name=name, + go_url=go_url, source_url=source_url, destination_folder=destination_folder, ) @@ -66,9 +70,12 @@ def _do_main(cli_args: list[str]): config_entries = [BuildConfigEntry.new_from_dictionary(item) for item in config_data] for entry in config_entries: + golang.install_go(workspace_path, entry.go_url, environment_label=entry.name) + build_environment = golang.acquire_environment(workspace_path, label=entry.name) + print(Rule(f"[bold yellow]{entry.name}")) source_parent_folder = do_download(workspace_path, entry) - cmd_node_folder = do_build(source_parent_folder) + cmd_node_folder = do_build(source_parent_folder, build_environment) copy_artifacts(cmd_node_folder, entry) @@ -77,11 +84,9 @@ def do_download(workspace: Path, entry: BuildConfigEntry) -> Path: extraction_folder = workspace / entry.name url = entry.source_url - print(f"Re-creating {download_folder} ...") shutil.rmtree(download_folder, ignore_errors=True) download_folder.mkdir(parents=True, exist_ok=True) - print(f"Re-creating {extraction_folder} ...") shutil.rmtree(extraction_folder, ignore_errors=True) extraction_folder.mkdir(parents=True, exist_ok=True) @@ -97,7 +102,7 @@ def do_download(workspace: Path, entry: BuildConfigEntry) -> Path: return extraction_folder -def do_build(source_parent_folder: Path) -> Path: +def do_build(source_parent_folder: Path, environment: golang.GoBuildEnvironment) -> Path: # If has one subfolder, that one is the source code subfolders = [Path(item.path) for item in os.scandir(source_parent_folder) if item.is_dir()] source_folder = subfolders[0] if len(subfolders) == 1 else source_parent_folder @@ -105,25 +110,21 @@ def do_build(source_parent_folder: Path) -> Path: cmd_node = source_folder / "cmd" / "node" go_mod = source_folder / "go.mod" - print(f"Building {cmd_node} ...") - - return_code = subprocess.check_call(["go", "build"], cwd=cmd_node) - if return_code != 0: - raise errors.KnownError(f"error code = {return_code}, see output") + golang.build(cmd_node, environment) + copy_wasmer_libraries(environment, go_mod, cmd_node) - copy_wasmer_libraries(go_mod, cmd_node) return cmd_node -def copy_wasmer_libraries(go_mod: Path, destination: Path): - go_path_variable = os.environ.get("GOPATH", "~/go") - go_path = Path(go_path_variable).expanduser().resolve() +def copy_wasmer_libraries(build_environment: golang.GoBuildEnvironment, go_mod: Path, destination: Path): + go_path = Path(build_environment.go_path).expanduser().resolve() vm_go_folder_name = get_chain_vm_go_folder_name(go_mod) vm_go_path = go_path / "pkg" / "mod" / vm_go_folder_name libraries = list((vm_go_path / "wasmer").glob("*.so")) + list((vm_go_path / "wasmer2").glob("*.so")) for library in libraries: shutil.copy(library, destination) + os.chmod(destination / library.name, FILE_MODE_NICE) diff --git a/multiversion/golang.py b/multiversion/golang.py new file mode 100644 index 0000000..abf967b --- /dev/null +++ b/multiversion/golang.py @@ -0,0 +1,72 @@ +import os +import shutil +import subprocess +import urllib.request +from pathlib import Path +from urllib.parse import urlparse + +from rich import print + +from multiversion import errors + + +class GoBuildEnvironment: + def __init__(self, system_path: str, go_path: str, go_cache: str, go_root: str) -> None: + self.system_path = system_path + self.go_path = go_path + self.go_cache = go_cache + self.go_root = go_root + + def to_dictionary(self) -> dict[str, str]: + return { + "PATH": self.system_path, + "GOPATH": self.go_path, + "GOCACHE": self.go_cache, + "GOROOT": self.go_root, + } + + +def acquire_environment(workspace: Path, label: str) -> GoBuildEnvironment: + directory = get_environment_directory(workspace, label) + current_system_path = os.environ.get("PATH", "") + + return GoBuildEnvironment( + system_path=f"{directory / 'go' / 'bin'}:{current_system_path}", + go_path=str(directory / "gopath"), + go_cache=str(directory / "gocache"), + go_root=str(directory / "go"), + ) + + +def get_environment_directory(workspace: Path, label: str) -> Path: + return workspace / f"go_{label}" + + +def install_go(workspace: Path, download_url: str, environment_label: str): + download_url_parsed = urlparse(download_url) + download_url_path = download_url_parsed.path + download_to_path = workspace / Path(download_url_path).name + environment_directory = get_environment_directory(workspace, environment_label) + + if environment_directory.exists(): + print(f"Go already installed in {environment_directory} ({environment_label}).") + return + + print(f"Downloading {download_url} to {download_to_path} ...") + urllib.request.urlretrieve(download_url, download_to_path) + + print(f"Extracting {environment_directory} ...") + shutil.rmtree(environment_directory, ignore_errors=True) + shutil.unpack_archive(download_to_path, environment_directory) + + print(f"Creating go environment directories ...") + (environment_directory / "gopath").mkdir() + (environment_directory / "gocache").mkdir() + + +def build(source_code: Path, environment: GoBuildEnvironment): + print(f"Building {source_code} ...") + + return_code = subprocess.check_call(["go", "build"], cwd=source_code, env=environment.to_dictionary()) + if return_code != 0: + raise errors.KnownError(f"error code = {return_code}, see output") From 54d9a638483b3e21a78f8741634d2d1a0b32cba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 1 Jul 2025 13:49:29 +0300 Subject: [PATCH 07/30] Logs refactoring. --- multiversion/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multiversion/build.py b/multiversion/build.py index 89dc76a..01708a1 100644 --- a/multiversion/build.py +++ b/multiversion/build.py @@ -70,10 +70,11 @@ def _do_main(cli_args: list[str]): config_entries = [BuildConfigEntry.new_from_dictionary(item) for item in config_data] for entry in config_entries: + print(Rule(f"[bold yellow]{entry.name}")) + golang.install_go(workspace_path, entry.go_url, environment_label=entry.name) build_environment = golang.acquire_environment(workspace_path, label=entry.name) - print(Rule(f"[bold yellow]{entry.name}")) source_parent_folder = do_download(workspace_path, entry) cmd_node_folder = do_build(source_parent_folder, build_environment) copy_artifacts(cmd_node_folder, entry) From 0bcc3bb525d2c707037e80e2c2ea560657ba7531 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 2 Jul 2025 13:42:08 +0300 Subject: [PATCH 08/30] Config, refactoring etc. --- multiversion/build.py | 32 +----------- multiversion/config.py | 111 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 31 deletions(-) create mode 100644 multiversion/config.py diff --git a/multiversion/build.py b/multiversion/build.py index 01708a1..4b2eb8a 100644 --- a/multiversion/build.py +++ b/multiversion/build.py @@ -13,40 +13,10 @@ from rich.rule import Rule from multiversion import errors, golang +from multiversion.config import BuildConfigEntry from multiversion.constants import FILE_MODE_NICE -class BuildConfigEntry: - def __init__(self, name: str, go_url: str, source_url: str, destination_folder: str) -> None: - if not name: - raise errors.KnownError("build 'name' is required") - if not go_url: - raise errors.KnownError("build 'go url' is required") - if not source_url: - raise errors.KnownError("build 'source' is required") - if not destination_folder: - raise errors.KnownError("build 'destination' is required") - - self.name = name - self.go_url = go_url - self.source_url = source_url - self.destination_folder = destination_folder - - @classmethod - def new_from_dictionary(cls, data: dict[str, Any]): - name = data.get("name") or "" - go_url = data.get("goUrl") or "" - source_url = data.get("sourceUrl") or "" - destination_folder = data.get("destinationFolder") or "" - - return cls( - name=name, - go_url=go_url, - source_url=source_url, - destination_folder=destination_folder, - ) - - def main(cli_args: list[str] = sys.argv[1:]): try: _do_main(cli_args) diff --git a/multiversion/config.py b/multiversion/config.py new file mode 100644 index 0000000..c2c566d --- /dev/null +++ b/multiversion/config.py @@ -0,0 +1,111 @@ + + +from pathlib import Path +from typing import Any + +from multiversion import errors + + +class BuildConfigEntry: + def __init__(self, name: str, go_url: str, source_url: str, destination_folder: str) -> None: + if not name: + raise errors.KnownError("build 'name' is required") + if not go_url: + raise errors.KnownError("build 'go url' is required") + if not source_url: + raise errors.KnownError("build 'source' is required") + if not destination_folder: + raise errors.KnownError("build 'destination' is required") + + self.name = name + self.go_url = go_url + self.source_url = source_url + self.destination_folder = destination_folder + + @classmethod + def new_from_dictionary(cls, data: dict[str, Any]): + name = data.get("name") or "" + go_url = data.get("goUrl") or "" + source_url = data.get("sourceUrl") or "" + destination_folder = data.get("destinationFolder") or "" + + return cls( + name=name, + go_url=go_url, + source_url=source_url, + destination_folder=destination_folder, + ) + + +class DriverConfig: + def __init__(self, shards: list[int], phases: list["DriverPhaseConfig"]) -> None: + if not shards: + raise errors.BadConfigurationError("'shards' are required") + if not phases: + raise errors.BadConfigurationError("'phases' are required") + + self.shards = shards + self.phases = phases + + @classmethod + def new_from_dictionary(cls, data: dict[str, Any]): + shards = data.get("shards") or [] + phases = data.get("phases") or [] + + return cls( + shards=shards, + phases=phases, + ) + + +class DriverPhaseConfig: + def __init__(self, + phase: str, + version: str, + until_epoch: int, + configuration_archive: str, + bin: str, + node_arguments: list[str], + with_db_lookup_extensions: bool, + with_indexing: bool) -> None: + if not phase: + raise errors.BadConfigurationError("'phase' is required") + if not version: + raise errors.BadConfigurationError("'version' is required") + if not until_epoch: + raise errors.BadConfigurationError("'until_epoch' is required") + if not configuration_archive: + raise errors.BadConfigurationError("'configuration_archive' is required") + if not bin: + raise errors.BadConfigurationError("'bin' is required") + + self.phase = phase + self.version = version + self.until_epoch = until_epoch + self.configuration_archive = configuration_archive + self.bin = Path(bin).expanduser().resolve() + self.node_arguments = node_arguments + self.with_db_lookup_extensions = with_db_lookup_extensions + self.with_indexing = with_indexing + + @classmethod + def new_from_dictionary(cls, data: dict[str, Any]): + phase = data.get("phase") or "" + version = data.get("version") or "" + until_epoch = data.get("untilEpoch") or 0 + configuration_archive = data.get("configurationArchive") or "" + bin = data.get("bin") or "" + node_arguments = data.get("nodeArguments") or [] + with_db_lookup_extensions = data.get("withDbLookupExtensions") or False + with_indexing = data.get("withIndexing") or False + + return cls( + phase=phase, + version=version, + until_epoch=until_epoch, + configuration_archive=configuration_archive, + bin=bin, + node_arguments=node_arguments, + with_db_lookup_extensions=with_db_lookup_extensions, + with_indexing=with_indexing, + ) From 3ad3c655a46ad493571f16a62d293deb1b0e7c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 2 Jul 2025 13:42:52 +0300 Subject: [PATCH 09/30] Refactor / rename. --- multiversion/build.py | 4 ++-- multiversion/golang.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/multiversion/build.py b/multiversion/build.py index 4b2eb8a..7b11c92 100644 --- a/multiversion/build.py +++ b/multiversion/build.py @@ -73,7 +73,7 @@ def do_download(workspace: Path, entry: BuildConfigEntry) -> Path: return extraction_folder -def do_build(source_parent_folder: Path, environment: golang.GoBuildEnvironment) -> Path: +def do_build(source_parent_folder: Path, environment: golang.BuildEnvironment) -> Path: # If has one subfolder, that one is the source code subfolders = [Path(item.path) for item in os.scandir(source_parent_folder) if item.is_dir()] source_folder = subfolders[0] if len(subfolders) == 1 else source_parent_folder @@ -87,7 +87,7 @@ def do_build(source_parent_folder: Path, environment: golang.GoBuildEnvironment) return cmd_node -def copy_wasmer_libraries(build_environment: golang.GoBuildEnvironment, go_mod: Path, destination: Path): +def copy_wasmer_libraries(build_environment: golang.BuildEnvironment, go_mod: Path, destination: Path): go_path = Path(build_environment.go_path).expanduser().resolve() vm_go_folder_name = get_chain_vm_go_folder_name(go_mod) vm_go_path = go_path / "pkg" / "mod" / vm_go_folder_name diff --git a/multiversion/golang.py b/multiversion/golang.py index abf967b..8a464ea 100644 --- a/multiversion/golang.py +++ b/multiversion/golang.py @@ -10,7 +10,7 @@ from multiversion import errors -class GoBuildEnvironment: +class BuildEnvironment: def __init__(self, system_path: str, go_path: str, go_cache: str, go_root: str) -> None: self.system_path = system_path self.go_path = go_path @@ -26,11 +26,11 @@ def to_dictionary(self) -> dict[str, str]: } -def acquire_environment(workspace: Path, label: str) -> GoBuildEnvironment: +def acquire_environment(workspace: Path, label: str) -> BuildEnvironment: directory = get_environment_directory(workspace, label) current_system_path = os.environ.get("PATH", "") - return GoBuildEnvironment( + return BuildEnvironment( system_path=f"{directory / 'go' / 'bin'}:{current_system_path}", go_path=str(directory / "gopath"), go_cache=str(directory / "gocache"), @@ -64,7 +64,7 @@ def install_go(workspace: Path, download_url: str, environment_label: str): (environment_directory / "gocache").mkdir() -def build(source_code: Path, environment: GoBuildEnvironment): +def build(source_code: Path, environment: BuildEnvironment): print(f"Building {source_code} ...") return_code = subprocess.check_call(["go", "build"], cwd=source_code, env=environment.to_dictionary()) From fbd50f207078b982301ad91ae98dd59288b3c5b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 2 Jul 2025 14:49:37 +0300 Subject: [PATCH 10/30] Multiversion -> multi-stage. Lanes etc. (sketch). --- {multiversion => multistage}/README.md | 8 +++++-- {multiversion => multistage}/build.json | 0 {multiversion => multistage}/build.py | 6 ++--- {multiversion => multistage}/config.py | 28 +++++++++++------------ {multiversion => multistage}/constants.py | 0 {multiversion => multistage}/driver.py | 16 +++++++++++-- multistage/driver.testnet.json | 16 +++++++++++++ {multiversion => multistage}/errors.py | 0 {multiversion => multistage}/golang.py | 2 +- multistage/node_controller.py | 2 ++ multistage/processing_lane.py | 7 ++++++ pyrightconfig.json | 2 +- 12 files changed, 64 insertions(+), 23 deletions(-) rename {multiversion => multistage}/README.md (73%) rename {multiversion => multistage}/build.json (100%) rename {multiversion => multistage}/build.py (96%) rename {multiversion => multistage}/config.py (85%) rename {multiversion => multistage}/constants.py (100%) rename {multiversion => multistage}/driver.py (50%) create mode 100644 multistage/driver.testnet.json rename {multiversion => multistage}/errors.py (100%) rename {multiversion => multistage}/golang.py (98%) create mode 100644 multistage/node_controller.py create mode 100644 multistage/processing_lane.py diff --git a/multiversion/README.md b/multistage/README.md similarity index 73% rename from multiversion/README.md rename to multistage/README.md index 28063bc..7859134 100644 --- a/multiversion/README.md +++ b/multistage/README.md @@ -18,8 +18,12 @@ pip install -r ./requirements.txt --upgrade Skip this flow if you choose to download the pre-built Node artifacts, instead of building them. -Go must be installed beforehand. +``` +PYTHONPATH=. python3 ./multistage/build.py --workspace=~/mvx-workspace --config=./multistage/build.json +``` + +## Set up an observer (or a squad) ``` -PYTHONPATH=. python3 ./multiversion/build.py --workspace=~/mvx-workspace --config=./multiversion/build.json +PYTHONPATH=. python3 ./multistage/driver.py --config=./multistage/driver.testnet.json ``` diff --git a/multiversion/build.json b/multistage/build.json similarity index 100% rename from multiversion/build.json rename to multistage/build.json diff --git a/multiversion/build.py b/multistage/build.py similarity index 96% rename from multiversion/build.py rename to multistage/build.py index 7b11c92..d5780e5 100644 --- a/multiversion/build.py +++ b/multistage/build.py @@ -12,9 +12,9 @@ from rich.panel import Panel from rich.rule import Rule -from multiversion import errors, golang -from multiversion.config import BuildConfigEntry -from multiversion.constants import FILE_MODE_NICE +from multistage import errors, golang +from multistage.config import BuildConfigEntry +from multistage.constants import FILE_MODE_NICE def main(cli_args: list[str] = sys.argv[1:]): diff --git a/multiversion/config.py b/multistage/config.py similarity index 85% rename from multiversion/config.py rename to multistage/config.py index c2c566d..da33483 100644 --- a/multiversion/config.py +++ b/multistage/config.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Any -from multiversion import errors +from multistage import errors class BuildConfigEntry: @@ -38,29 +38,29 @@ def new_from_dictionary(cls, data: dict[str, Any]): class DriverConfig: - def __init__(self, shards: list[int], phases: list["DriverPhaseConfig"]) -> None: + def __init__(self, shards: list[int], stages: list["StageConfig"]) -> None: if not shards: raise errors.BadConfigurationError("'shards' are required") - if not phases: - raise errors.BadConfigurationError("'phases' are required") + if not stages: + raise errors.BadConfigurationError("'stages' are required") self.shards = shards - self.phases = phases + self.stages = stages @classmethod def new_from_dictionary(cls, data: dict[str, Any]): shards = data.get("shards") or [] - phases = data.get("phases") or [] + stages = data.get("stages") or [] return cls( shards=shards, - phases=phases, + stages=stages, ) -class DriverPhaseConfig: +class StageConfig: def __init__(self, - phase: str, + name: str, version: str, until_epoch: int, configuration_archive: str, @@ -68,8 +68,8 @@ def __init__(self, node_arguments: list[str], with_db_lookup_extensions: bool, with_indexing: bool) -> None: - if not phase: - raise errors.BadConfigurationError("'phase' is required") + if not name: + raise errors.BadConfigurationError("'name' is required") if not version: raise errors.BadConfigurationError("'version' is required") if not until_epoch: @@ -79,7 +79,7 @@ def __init__(self, if not bin: raise errors.BadConfigurationError("'bin' is required") - self.phase = phase + self.name = name self.version = version self.until_epoch = until_epoch self.configuration_archive = configuration_archive @@ -90,7 +90,7 @@ def __init__(self, @classmethod def new_from_dictionary(cls, data: dict[str, Any]): - phase = data.get("phase") or "" + name = data.get("name") or "" version = data.get("version") or "" until_epoch = data.get("untilEpoch") or 0 configuration_archive = data.get("configurationArchive") or "" @@ -100,7 +100,7 @@ def new_from_dictionary(cls, data: dict[str, Any]): with_indexing = data.get("withIndexing") or False return cls( - phase=phase, + name=name, version=version, until_epoch=until_epoch, configuration_archive=configuration_archive, diff --git a/multiversion/constants.py b/multistage/constants.py similarity index 100% rename from multiversion/constants.py rename to multistage/constants.py diff --git a/multiversion/driver.py b/multistage/driver.py similarity index 50% rename from multiversion/driver.py rename to multistage/driver.py index f16b99c..1533a55 100644 --- a/multiversion/driver.py +++ b/multistage/driver.py @@ -1,11 +1,16 @@ +import json import sys import traceback from argparse import ArgumentParser +from pathlib import Path +from typing import Any from rich import print from rich.panel import Panel +from rich.rule import Rule -from multiversion import errors +from multistage import errors +from multistage.config import DriverConfig def main(cli_args: list[str] = sys.argv[1:]): @@ -19,10 +24,17 @@ def main(cli_args: list[str] = sys.argv[1:]): def _do_main(cli_args: list[str]): parser = ArgumentParser() + parser.add_argument("--config", required=True, help="path of the 'driver' configuration file") args = parser.parse_args(cli_args) + config_path = Path(args.config).expanduser().resolve() + config_data = json.loads(config_path.read_text()) + driver_config = DriverConfig.new_from_dictionary(config_data) + + for stage in driver_config.stages: + print(Rule(f"[bold yellow]{stage.name}")) + if __name__ == "__main__": ret = main(sys.argv[1:]) sys.exit(ret) - diff --git a/multistage/driver.testnet.json b/multistage/driver.testnet.json new file mode 100644 index 0000000..39bb96c --- /dev/null +++ b/multistage/driver.testnet.json @@ -0,0 +1,16 @@ +{ + "shards": [0, 1, 2, 4294967295], + "nodesRootPath": "~/mvx-nodes", + "stages": [ + { + "name": "1st (Sirius)", + "version": "Sirius", + "untilEpoch": 42, + "configurationArchive": "https://github.com/multiversx/mx-chain-mainnet-config/releases/tag/v1.6.18.0.zip", + "bin": "~/mvx-binaries/sirius", + "nodeArguments": ["--log-save", "--log-level", "*:DEBUG"], + "withDbLookupExtensions": true, + "withIndexing": false + } + ] +} diff --git a/multiversion/errors.py b/multistage/errors.py similarity index 100% rename from multiversion/errors.py rename to multistage/errors.py diff --git a/multiversion/golang.py b/multistage/golang.py similarity index 98% rename from multiversion/golang.py rename to multistage/golang.py index 8a464ea..613de86 100644 --- a/multiversion/golang.py +++ b/multistage/golang.py @@ -7,7 +7,7 @@ from rich import print -from multiversion import errors +from multistage import errors class BuildEnvironment: diff --git a/multistage/node_controller.py b/multistage/node_controller.py new file mode 100644 index 0000000..31931b5 --- /dev/null +++ b/multistage/node_controller.py @@ -0,0 +1,2 @@ +class NodeController: + pass diff --git a/multistage/processing_lane.py b/multistage/processing_lane.py new file mode 100644 index 0000000..15fdc88 --- /dev/null +++ b/multistage/processing_lane.py @@ -0,0 +1,7 @@ +from multistage.config import StageConfig + + +class ProcessingLane: + def __init__(self, shard: int, stages: list[StageConfig]) -> None: + self.shard = shard + self.stages = stages diff --git a/pyrightconfig.json b/pyrightconfig.json index 72a3c50..8c9133c 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,5 +1,5 @@ { - "include": ["multiversion"], + "include": ["multistage"], "exclude": ["**/__pycache__"], "ignore": [], "defineConstant": { From 02897fd80046c0e7eebf243861ca10362f619f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 2 Jul 2025 15:55:26 +0300 Subject: [PATCH 11/30] Adjust config. Sketch coroutines logic / lanes. --- multistage/config.py | 39 +++++++++++----------- multistage/driver.py | 7 ++-- multistage/driver.testnet.json | 61 ++++++++++++++++++++++++++++------ multistage/node_controller.py | 14 +++++++- multistage/processing_lane.py | 56 +++++++++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 33 deletions(-) diff --git a/multistage/config.py b/multistage/config.py index da33483..aa5085d 100644 --- a/multistage/config.py +++ b/multistage/config.py @@ -38,31 +38,33 @@ def new_from_dictionary(cls, data: dict[str, Any]): class DriverConfig: - def __init__(self, shards: list[int], stages: list["StageConfig"]) -> None: - if not shards: - raise errors.BadConfigurationError("'shards' are required") - if not stages: - raise errors.BadConfigurationError("'stages' are required") + def __init__(self, lanes: list["LaneConfig"]) -> None: + if not lanes: + raise errors.BadConfigurationError("'lanes' are required") - self.shards = shards - self.stages = stages + self.lanes = lanes @classmethod def new_from_dictionary(cls, data: dict[str, Any]): - shards = data.get("shards") or [] - stages = data.get("stages") or [] + lanes = data.get("lanes") or [] return cls( - shards=shards, - stages=stages, + lanes=lanes, ) +class LaneConfig: + def __init__(self, name: str, working_directory: str, stages: list["StageConfig"]) -> None: + self.name = name + self.working_directory = working_directory + self.stages = stages + + class StageConfig: def __init__(self, name: str, - version: str, until_epoch: int, + node_status_url: str, configuration_archive: str, bin: str, node_arguments: list[str], @@ -70,17 +72,16 @@ def __init__(self, with_indexing: bool) -> None: if not name: raise errors.BadConfigurationError("'name' is required") - if not version: - raise errors.BadConfigurationError("'version' is required") if not until_epoch: - raise errors.BadConfigurationError("'until_epoch' is required") + raise errors.BadConfigurationError("'until epoch' is required") + if not node_status_url: + raise errors.BadConfigurationError("'node status url' is required") if not configuration_archive: - raise errors.BadConfigurationError("'configuration_archive' is required") + raise errors.BadConfigurationError("'configuration archive' is required") if not bin: raise errors.BadConfigurationError("'bin' is required") self.name = name - self.version = version self.until_epoch = until_epoch self.configuration_archive = configuration_archive self.bin = Path(bin).expanduser().resolve() @@ -91,8 +92,8 @@ def __init__(self, @classmethod def new_from_dictionary(cls, data: dict[str, Any]): name = data.get("name") or "" - version = data.get("version") or "" until_epoch = data.get("untilEpoch") or 0 + node_status_url = data.get("nodeStatusUrl") or "" configuration_archive = data.get("configurationArchive") or "" bin = data.get("bin") or "" node_arguments = data.get("nodeArguments") or [] @@ -101,8 +102,8 @@ def new_from_dictionary(cls, data: dict[str, Any]): return cls( name=name, - version=version, until_epoch=until_epoch, + node_status_url=node_status_url, configuration_archive=configuration_archive, bin=bin, node_arguments=node_arguments, diff --git a/multistage/driver.py b/multistage/driver.py index 1533a55..256d40a 100644 --- a/multistage/driver.py +++ b/multistage/driver.py @@ -31,8 +31,11 @@ def _do_main(cli_args: list[str]): config_data = json.loads(config_path.read_text()) driver_config = DriverConfig.new_from_dictionary(config_data) - for stage in driver_config.stages: - print(Rule(f"[bold yellow]{stage.name}")) + for lane in driver_config.lanes: + print(Rule(f"[bold yellow]{lane.name}")) + + for stage in lane.stages: + print(Rule(f"[bold yellow]{stage.name}")) if __name__ == "__main__": diff --git a/multistage/driver.testnet.json b/multistage/driver.testnet.json index 39bb96c..25ff002 100644 --- a/multistage/driver.testnet.json +++ b/multistage/driver.testnet.json @@ -1,16 +1,55 @@ { - "shards": [0, 1, 2, 4294967295], - "nodesRootPath": "~/mvx-nodes", - "stages": [ + "lanes": [ { - "name": "1st (Sirius)", - "version": "Sirius", - "untilEpoch": 42, - "configurationArchive": "https://github.com/multiversx/mx-chain-mainnet-config/releases/tag/v1.6.18.0.zip", - "bin": "~/mvx-binaries/sirius", - "nodeArguments": ["--log-save", "--log-level", "*:DEBUG"], - "withDbLookupExtensions": true, - "withIndexing": false + "name": "Shard 0", + "workingDirectory": "~/mvx-nodes/shard-0", + "stages": [ + { + "name": "Spica", + "untilEpoch": 201, + "nodeStatusUrl": "http://localhost:8080/node/status", + "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/releases/tag/T1.8.12.1.zip", + "bin": "~/mvx-binaries/spica", + "nodeArguments": [ + "--log-save", + "--log-level=*:DEBUG", + "--destination-shard-as-observer=0", + "--rest-api-interface=8080" + ], + "withDbLookupExtensions": true, + "withIndexing": false + }, + { + "name": "Andromeda", + "untilEpoch": 767, + "nodeStatusUrl": "http://localhost:8080/node/status", + "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/releases/tag/T1.9.6.0.zip", + "bin": "~/mvx-binaries/andromeda", + "nodeArguments": [ + "--log-save", + "--log-level=*:DEBUG", + "--destination-shard-as-observer=0", + "--rest-api-interface=8080" + ], + "withDbLookupExtensions": true, + "withIndexing": false + }, + { + "name": "Barnard", + "untilEpoch": 4294967295, + "nodeStatusUrl": "http://localhost:8080/node/status", + "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/releases/tag/T1.10.1.0.zip", + "bin": "~/mvx-binaries/barnard", + "nodeArguments": [ + "--log-save", + "--log-level=*:DEBUG", + "--destination-shard-as-observer=0", + "--rest-api-interface=8080" + ], + "withDbLookupExtensions": true, + "withIndexing": false + } + ] } ] } diff --git a/multistage/node_controller.py b/multistage/node_controller.py index 31931b5..496544a 100644 --- a/multistage/node_controller.py +++ b/multistage/node_controller.py @@ -1,2 +1,14 @@ +from pathlib import Path + + class NodeController: - pass + def __init__(self, shard: int, api_port: int) -> None: + self.pid = 0 + self.shard = shard + self.api_port = api_port + + async def start(self, args: list[str], cwd: Path, delay: int = 0): + pass + + def stop(self): + pass diff --git a/multistage/processing_lane.py b/multistage/processing_lane.py index 15fdc88..0c68aba 100644 --- a/multistage/processing_lane.py +++ b/multistage/processing_lane.py @@ -1,3 +1,8 @@ +import asyncio +import os +from pathlib import Path +from typing import Any, Coroutine + from multistage.config import StageConfig @@ -5,3 +10,54 @@ class ProcessingLane: def __init__(self, shard: int, stages: list[StageConfig]) -> None: self.shard = shard self.stages = stages + + +def start(): + try: + loop = asyncio.get_event_loop() + loop.run_until_complete(do_start()) + loop.close() + asyncio.set_event_loop(asyncio.new_event_loop()) + except KeyboardInterrupt: + pass + + +async def do_start(): + coroutines: list[Coroutine[Any, Any, None]] = [ + run( + ["TBD", "TBD"], + cwd=Path("."), + delay=1000, + ), + monitor_network() + ] + + tasks = [asyncio.create_task(item) for item in coroutines] + await asyncio.gather(*tasks) + + +async def monitor_network(): + loop = asyncio.get_running_loop() + + while True: + await asyncio.sleep(1000) + + +async def run(args: list[str], cwd: Path, delay: int = 0): + await asyncio.sleep(delay) + + print(f"Starting process {args} in folder {cwd}") + + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = str(cwd) + + process = await asyncio.create_subprocess_exec( + *args, + cwd=cwd, + limit=1024 * 512, + env=env, + ) + + pid = process.pid + return_code = await process.wait() + print(f"Proces [{pid}] stopped. Return code: {return_code}.") From 00b3c9cdf52289ba8a21dcaca5fd4fed22521f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 3 Jul 2025 15:36:20 +0300 Subject: [PATCH 12/30] Continue working on driver etc. --- multistage/README.md | 2 +- multistage/config.py | 44 ++++++++++++++++++++---- multistage/driver.py | 21 +++++++++--- multistage/driver.testnet.json | 8 ++--- multistage/processing_lane.py | 63 +++++++++++++++++----------------- 5 files changed, 91 insertions(+), 47 deletions(-) diff --git a/multistage/README.md b/multistage/README.md index 7859134..25829e6 100644 --- a/multistage/README.md +++ b/multistage/README.md @@ -25,5 +25,5 @@ PYTHONPATH=. python3 ./multistage/build.py --workspace=~/mvx-workspace --config= ## Set up an observer (or a squad) ``` -PYTHONPATH=. python3 ./multistage/driver.py --config=./multistage/driver.testnet.json +PYTHONPATH=. python3 ./multistage/driver.py --config=./multistage/driver.testnet.json --lane=shard_0 --stage=spica ``` diff --git a/multistage/config.py b/multistage/config.py index aa5085d..bf86f4d 100644 --- a/multistage/config.py +++ b/multistage/config.py @@ -43,21 +43,53 @@ def __init__(self, lanes: list["LaneConfig"]) -> None: raise errors.BadConfigurationError("'lanes' are required") self.lanes = lanes + self.lanes_by_name = {lane.name: lane for lane in lanes} @classmethod def new_from_dictionary(cls, data: dict[str, Any]): - lanes = data.get("lanes") or [] + lanes_records = data.get("lanes") or [] + lanes = [LaneConfig.new_from_dictionary(record) for record in lanes_records] return cls( lanes=lanes, ) + def get_lanes_names(self) -> list[str]: + return [lane.name for lane in self.lanes] + + def get_lane(self, name: str) -> "LaneConfig": + return self.lanes_by_name[name] + class LaneConfig: def __init__(self, name: str, working_directory: str, stages: list["StageConfig"]) -> None: + if not name: + raise errors.BadConfigurationError("for all lanes, 'name' is required") + if not working_directory: + raise errors.BadConfigurationError(f"for lane {name}, 'working directory' is required") + if not stages: + raise errors.BadConfigurationError(f"for lane {name}, 'stages' are required") + self.name = name self.working_directory = working_directory self.stages = stages + self.stages_by_name = {stage.name: stage for stage in stages} + + @classmethod + def new_from_dictionary(cls, data: dict[str, Any]): + name = data.get("name") or "" + working_directory = data.get("workingDirectory") or "" + stages_records = data.get("stages") or [] + stages = [StageConfig.new_from_dictionary(record) for record in stages_records] + + return cls( + name=name, + working_directory=working_directory, + stages=stages, + ) + + def get_stages_names(self) -> list[str]: + return [stage.name for stage in self.stages] class StageConfig: @@ -71,15 +103,15 @@ def __init__(self, with_db_lookup_extensions: bool, with_indexing: bool) -> None: if not name: - raise errors.BadConfigurationError("'name' is required") + raise errors.BadConfigurationError("for all stages, 'name' is required") if not until_epoch: - raise errors.BadConfigurationError("'until epoch' is required") + raise errors.BadConfigurationError(f"for stage {name}, 'until epoch' is required") if not node_status_url: - raise errors.BadConfigurationError("'node status url' is required") + raise errors.BadConfigurationError(f"for stage {name}, 'node status url' is required") if not configuration_archive: - raise errors.BadConfigurationError("'configuration archive' is required") + raise errors.BadConfigurationError(f"for stage {name}, 'configuration archive' is required") if not bin: - raise errors.BadConfigurationError("'bin' is required") + raise errors.BadConfigurationError(f"for stage {name}, 'bin' is required") self.name = name self.until_epoch = until_epoch diff --git a/multistage/driver.py b/multistage/driver.py index 256d40a..c860946 100644 --- a/multistage/driver.py +++ b/multistage/driver.py @@ -7,10 +7,12 @@ from rich import print from rich.panel import Panel +from rich.prompt import Prompt from rich.rule import Rule from multistage import errors from multistage.config import DriverConfig +from multistage.processing_lane import ProcessingLane def main(cli_args: list[str] = sys.argv[1:]): @@ -25,17 +27,28 @@ def main(cli_args: list[str] = sys.argv[1:]): def _do_main(cli_args: list[str]): parser = ArgumentParser() parser.add_argument("--config", required=True, help="path of the 'driver' configuration file") + parser.add_argument("--lane", required=True, help="which lane to handle") + parser.add_argument("--stage", required=True, help="initial stage on the lane") args = parser.parse_args(cli_args) config_path = Path(args.config).expanduser().resolve() config_data = json.loads(config_path.read_text()) driver_config = DriverConfig.new_from_dictionary(config_data) + lane_name = args.lane + stage_name = args.stage - for lane in driver_config.lanes: - print(Rule(f"[bold yellow]{lane.name}")) + if lane_name not in driver_config.get_lanes_names(): + raise errors.BadConfigurationError(f"unknown lane: {lane_name}") - for stage in lane.stages: - print(Rule(f"[bold yellow]{stage.name}")) + lane_config = driver_config.get_lane(lane_name) + + if stage_name not in lane_config.get_stages_names(): + raise errors.BadConfigurationError(f"unknown stage: {stage_name}") + + print(f"[bold yellow]Lane: {lane_name}") + print(f"[bold yellow]Initial stage: {stage_name}") + + lane = ProcessingLane(lane_config) if __name__ == "__main__": diff --git a/multistage/driver.testnet.json b/multistage/driver.testnet.json index 25ff002..adee25c 100644 --- a/multistage/driver.testnet.json +++ b/multistage/driver.testnet.json @@ -1,11 +1,11 @@ { "lanes": [ { - "name": "Shard 0", + "name": "shard_0", "workingDirectory": "~/mvx-nodes/shard-0", "stages": [ { - "name": "Spica", + "name": "spica", "untilEpoch": 201, "nodeStatusUrl": "http://localhost:8080/node/status", "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/releases/tag/T1.8.12.1.zip", @@ -20,7 +20,7 @@ "withIndexing": false }, { - "name": "Andromeda", + "name": "andromeda", "untilEpoch": 767, "nodeStatusUrl": "http://localhost:8080/node/status", "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/releases/tag/T1.9.6.0.zip", @@ -35,7 +35,7 @@ "withIndexing": false }, { - "name": "Barnard", + "name": "barnard", "untilEpoch": 4294967295, "nodeStatusUrl": "http://localhost:8080/node/status", "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/releases/tag/T1.10.1.0.zip", diff --git a/multistage/processing_lane.py b/multistage/processing_lane.py index 0c68aba..df1e16c 100644 --- a/multistage/processing_lane.py +++ b/multistage/processing_lane.py @@ -1,39 +1,40 @@ import asyncio import os from pathlib import Path -from typing import Any, Coroutine +from typing import Any, Coroutine, Optional -from multistage.config import StageConfig +from multistage.config import LaneConfig, StageConfig +from multistage.node_controller import NodeController class ProcessingLane: - def __init__(self, shard: int, stages: list[StageConfig]) -> None: - self.shard = shard - self.stages = stages - - -def start(): - try: - loop = asyncio.get_event_loop() - loop.run_until_complete(do_start()) - loop.close() - asyncio.set_event_loop(asyncio.new_event_loop()) - except KeyboardInterrupt: - pass - - -async def do_start(): - coroutines: list[Coroutine[Any, Any, None]] = [ - run( - ["TBD", "TBD"], - cwd=Path("."), - delay=1000, - ), - monitor_network() - ] - - tasks = [asyncio.create_task(item) for item in coroutines] - await asyncio.gather(*tasks) + def __init__(self, config: LaneConfig) -> None: + self.config = config + self.current_stage_index = 0 + self.current_node_controller: Optional[NodeController] = None + + def start(self): + try: + loop = asyncio.get_event_loop() + loop.run_until_complete(self._do_start()) + loop.close() + asyncio.set_event_loop(asyncio.new_event_loop()) + except KeyboardInterrupt: + print("Processing lange interrupted.") + + async def _do_start(self): + # get + + coroutines: list[Coroutine[Any, Any, None]] = [ + run( + args=["TBD", "TBD"], + cwd=Path("."), + ), + monitor_network() + ] + + tasks = [asyncio.create_task(item) for item in coroutines] + await asyncio.gather(*tasks) async def monitor_network(): @@ -43,9 +44,7 @@ async def monitor_network(): await asyncio.sleep(1000) -async def run(args: list[str], cwd: Path, delay: int = 0): - await asyncio.sleep(delay) - +async def run(args: list[str], cwd: Path): print(f"Starting process {args} in folder {cwd}") env = os.environ.copy() From da41127779823f55dbf1feb675acca5b5178fdfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 3 Jul 2025 16:04:50 +0300 Subject: [PATCH 13/30] Lanes, node controller, work in progress. --- multistage/config.py | 8 +++++++ multistage/constants.py | 1 + multistage/driver.py | 11 +++++---- multistage/node_controller.py | 41 ++++++++++++++++++++++++++------ multistage/processing_lane.py | 44 ++++++++++------------------------- 5 files changed, 61 insertions(+), 44 deletions(-) diff --git a/multistage/config.py b/multistage/config.py index bf86f4d..6893ac9 100644 --- a/multistage/config.py +++ b/multistage/config.py @@ -39,8 +39,12 @@ def new_from_dictionary(cls, data: dict[str, Any]): class DriverConfig: def __init__(self, lanes: list["LaneConfig"]) -> None: + lanes_names = [lane.name for lane in lanes] + if not lanes: raise errors.BadConfigurationError("'lanes' are required") + if len(lanes_names) > len(set(lanes_names)): + raise errors.BadConfigurationError("lanes names must be unique") self.lanes = lanes self.lanes_by_name = {lane.name: lane for lane in lanes} @@ -63,12 +67,16 @@ def get_lane(self, name: str) -> "LaneConfig": class LaneConfig: def __init__(self, name: str, working_directory: str, stages: list["StageConfig"]) -> None: + stages_names = [stage.name for stage in stages] + if not name: raise errors.BadConfigurationError("for all lanes, 'name' is required") if not working_directory: raise errors.BadConfigurationError(f"for lane {name}, 'working directory' is required") if not stages: raise errors.BadConfigurationError(f"for lane {name}, 'stages' are required") + if len(stages) > len(set(stages)): + raise errors.BadConfigurationError("stages names must be unique") self.name = name self.working_directory = working_directory diff --git a/multistage/constants.py b/multistage/constants.py index a791d12..ce4d2d3 100644 --- a/multistage/constants.py +++ b/multistage/constants.py @@ -1,6 +1,7 @@ import stat METACHAIN_ID = 4294967295 +NODE_PROCESS_ULIMIT = 1024 * 512 # Read, write and execute by owner, read and execute by group and others FILE_MODE_NICE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH diff --git a/multistage/driver.py b/multistage/driver.py index c860946..47b412e 100644 --- a/multistage/driver.py +++ b/multistage/driver.py @@ -35,20 +35,21 @@ def _do_main(cli_args: list[str]): config_data = json.loads(config_path.read_text()) driver_config = DriverConfig.new_from_dictionary(config_data) lane_name = args.lane - stage_name = args.stage + initial_stage_name = args.stage if lane_name not in driver_config.get_lanes_names(): raise errors.BadConfigurationError(f"unknown lane: {lane_name}") lane_config = driver_config.get_lane(lane_name) - if stage_name not in lane_config.get_stages_names(): - raise errors.BadConfigurationError(f"unknown stage: {stage_name}") + if initial_stage_name not in lane_config.get_stages_names(): + raise errors.BadConfigurationError(f"unknown stage: {initial_stage_name}") print(f"[bold yellow]Lane: {lane_name}") - print(f"[bold yellow]Initial stage: {stage_name}") + print(f"[bold yellow]Initial stage: {initial_stage_name}") - lane = ProcessingLane(lane_config) + lane = ProcessingLane(lane_config, initial_stage_name) + lane.start() if __name__ == "__main__": diff --git a/multistage/node_controller.py b/multistage/node_controller.py index 496544a..16ec57a 100644 --- a/multistage/node_controller.py +++ b/multistage/node_controller.py @@ -1,14 +1,41 @@ +import asyncio +import os +from asyncio.subprocess import Process from pathlib import Path +from typing import Optional + +from rich import print + +from multistage.constants import NODE_PROCESS_ULIMIT class NodeController: - def __init__(self, shard: int, api_port: int) -> None: - self.pid = 0 - self.shard = shard - self.api_port = api_port + def __init__(self) -> None: + self.process: Optional[Process] = None + + async def start(self, program: str, args: list[str], cwd: Path): + print(f"Starting node ...") + print("args:", args) + print("cwd:", cwd) + + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = str(cwd) - async def start(self, args: list[str], cwd: Path, delay: int = 0): - pass + self.process = await asyncio.create_subprocess_exec( + program, + *args, + cwd=cwd, + limit=NODE_PROCESS_ULIMIT, + env=env, + ) + + return_code = await self.process.wait() + print(f"Proces [{self.process.pid}] stopped. Return code: {return_code}.") + + self.process = None def stop(self): - pass + assert self.process is not None + + print("Stopping node ...") + self.process.kill() diff --git a/multistage/processing_lane.py b/multistage/processing_lane.py index df1e16c..c54d95b 100644 --- a/multistage/processing_lane.py +++ b/multistage/processing_lane.py @@ -3,12 +3,14 @@ from pathlib import Path from typing import Any, Coroutine, Optional +from rich import print + from multistage.config import LaneConfig, StageConfig from multistage.node_controller import NodeController class ProcessingLane: - def __init__(self, config: LaneConfig) -> None: + def __init__(self, config: LaneConfig, initial_stage_name: str) -> None: self.config = config self.current_stage_index = 0 self.current_node_controller: Optional[NodeController] = None @@ -20,43 +22,21 @@ def start(self): loop.close() asyncio.set_event_loop(asyncio.new_event_loop()) except KeyboardInterrupt: - print("Processing lange interrupted.") + print("Processing lane interrupted.") async def _do_start(self): - # get + node_controller = NodeController() coroutines: list[Coroutine[Any, Any, None]] = [ - run( - args=["TBD", "TBD"], - cwd=Path("."), - ), - monitor_network() + node_controller.start("tbd", [], Path("")), + self.monitor_node() ] tasks = [asyncio.create_task(item) for item in coroutines] - await asyncio.gather(*tasks) - - -async def monitor_network(): - loop = asyncio.get_running_loop() - - while True: - await asyncio.sleep(1000) - - -async def run(args: list[str], cwd: Path): - print(f"Starting process {args} in folder {cwd}") - - env = os.environ.copy() - env["LD_LIBRARY_PATH"] = str(cwd) + await asyncio.gather(*tasks, return_exceptions=False) - process = await asyncio.create_subprocess_exec( - *args, - cwd=cwd, - limit=1024 * 512, - env=env, - ) + async def monitor_node(self): + loop = asyncio.get_running_loop() - pid = process.pid - return_code = await process.wait() - print(f"Proces [{pid}] stopped. Return code: {return_code}.") + while True: + await asyncio.sleep(1000) From 1b1b0d31d430eaff8585e08554efec3e6908606e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Thu, 3 Jul 2025 16:50:24 +0300 Subject: [PATCH 14/30] Handle node's stages, work in progress. --- multistage/config.py | 6 ++++++ multistage/node_controller.py | 6 ++++++ multistage/processing_lane.py | 34 ++++++++++++++++++++++++---------- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/multistage/config.py b/multistage/config.py index 6893ac9..d87e149 100644 --- a/multistage/config.py +++ b/multistage/config.py @@ -99,6 +99,11 @@ def new_from_dictionary(cls, data: dict[str, Any]): def get_stages_names(self) -> list[str]: return [stage.name for stage in self.stages] + def get_stages_including_and_after(self, initial_stage_name: str) -> list["StageConfig"]: + stages_names = self.get_stages_names() + index_of_initial_stage_name = stages_names.index(initial_stage_name) + return self.stages[index_of_initial_stage_name:] + class StageConfig: def __init__(self, @@ -123,6 +128,7 @@ def __init__(self, self.name = name self.until_epoch = until_epoch + self.node_status_url = node_status_url self.configuration_archive = configuration_archive self.bin = Path(bin).expanduser().resolve() self.node_arguments = node_arguments diff --git a/multistage/node_controller.py b/multistage/node_controller.py index 16ec57a..42a9dad 100644 --- a/multistage/node_controller.py +++ b/multistage/node_controller.py @@ -12,6 +12,7 @@ class NodeController: def __init__(self) -> None: self.process: Optional[Process] = None + self.return_code = -1 async def start(self, program: str, args: list[str], cwd: Path): print(f"Starting node ...") @@ -33,9 +34,14 @@ async def start(self, program: str, args: list[str], cwd: Path): print(f"Proces [{self.process.pid}] stopped. Return code: {return_code}.") self.process = None + self.return_code = return_code def stop(self): assert self.process is not None print("Stopping node ...") self.process.kill() + self.process = None + + def is_running(self) -> bool: + return self.process is not None diff --git a/multistage/processing_lane.py b/multistage/processing_lane.py index c54d95b..624bfa9 100644 --- a/multistage/processing_lane.py +++ b/multistage/processing_lane.py @@ -4,6 +4,7 @@ from typing import Any, Coroutine, Optional from rich import print +from rich.rule import Rule from multistage.config import LaneConfig, StageConfig from multistage.node_controller import NodeController @@ -12,7 +13,8 @@ class ProcessingLane: def __init__(self, config: LaneConfig, initial_stage_name: str) -> None: self.config = config - self.current_stage_index = 0 + self.initial_stage_name = initial_stage_name + self.current_stage: Optional[StageConfig] = None self.current_node_controller: Optional[NodeController] = None def start(self): @@ -25,18 +27,30 @@ def start(self): print("Processing lane interrupted.") async def _do_start(self): - node_controller = NodeController() + stages = self.config.get_stages_including_and_after(self.initial_stage_name) - coroutines: list[Coroutine[Any, Any, None]] = [ - node_controller.start("tbd", [], Path("")), - self.monitor_node() - ] + for stage in stages: + print(Rule(f"[bold yellow]{stage.name}")) - tasks = [asyncio.create_task(item) for item in coroutines] - await asyncio.gather(*tasks, return_exceptions=False) + self.current_stage = stage + self.current_node_controller = NodeController() + + coroutines: list[Coroutine[Any, Any, None]] = [ + # TBD: change this, start node. + self.current_node_controller.start("ls", [], Path("")), + self.monitor_node() + ] + + tasks = [asyncio.create_task(item) for item in coroutines] + await asyncio.gather(*tasks, return_exceptions=False) async def monitor_node(self): - loop = asyncio.get_running_loop() + assert self.current_stage is not None + + print("Node status URL:", self.current_stage.node_status_url) while True: - await asyncio.sleep(1000) + await asyncio.sleep(1) + + if self.current_node_controller and not self.current_node_controller.is_running(): + return From 89d37da03dfaa9ab2d913ee45349269e47f04e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 4 Jul 2025 10:25:32 +0300 Subject: [PATCH 15/30] Lanes, starting phases etc. --- multistage/config.py | 2 +- multistage/constants.py | 2 ++ multistage/driver.py | 4 ++-- ...{processing_lane.py => lane_controller.py} | 24 +++++++++++++------ 4 files changed, 22 insertions(+), 10 deletions(-) rename multistage/{processing_lane.py => lane_controller.py} (67%) diff --git a/multistage/config.py b/multistage/config.py index d87e149..14f99ed 100644 --- a/multistage/config.py +++ b/multistage/config.py @@ -79,7 +79,7 @@ def __init__(self, name: str, working_directory: str, stages: list["StageConfig" raise errors.BadConfigurationError("stages names must be unique") self.name = name - self.working_directory = working_directory + self.working_directory = Path(working_directory).expanduser().resolve() self.stages = stages self.stages_by_name = {stage.name: stage for stage in stages} diff --git a/multistage/constants.py b/multistage/constants.py index ce4d2d3..f990635 100644 --- a/multistage/constants.py +++ b/multistage/constants.py @@ -2,6 +2,8 @@ METACHAIN_ID = 4294967295 NODE_PROCESS_ULIMIT = 1024 * 512 +NODE_MONITORING_PERIOD = 1 +NODE_RETURN_CODE_SUCCESS = 0 # Read, write and execute by owner, read and execute by group and others FILE_MODE_NICE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH diff --git a/multistage/driver.py b/multistage/driver.py index 47b412e..7ed2a06 100644 --- a/multistage/driver.py +++ b/multistage/driver.py @@ -12,7 +12,7 @@ from multistage import errors from multistage.config import DriverConfig -from multistage.processing_lane import ProcessingLane +from multistage.lane_controller import LaneController def main(cli_args: list[str] = sys.argv[1:]): @@ -48,7 +48,7 @@ def _do_main(cli_args: list[str]): print(f"[bold yellow]Lane: {lane_name}") print(f"[bold yellow]Initial stage: {initial_stage_name}") - lane = ProcessingLane(lane_config, initial_stage_name) + lane = LaneController(lane_config, initial_stage_name) lane.start() diff --git a/multistage/processing_lane.py b/multistage/lane_controller.py similarity index 67% rename from multistage/processing_lane.py rename to multistage/lane_controller.py index 624bfa9..322c1fb 100644 --- a/multistage/processing_lane.py +++ b/multistage/lane_controller.py @@ -1,16 +1,16 @@ import asyncio -import os -from pathlib import Path from typing import Any, Coroutine, Optional from rich import print from rich.rule import Rule from multistage.config import LaneConfig, StageConfig +from multistage.constants import (NODE_MONITORING_PERIOD, + NODE_RETURN_CODE_SUCCESS) from multistage.node_controller import NodeController -class ProcessingLane: +class LaneController: def __init__(self, config: LaneConfig, initial_stage_name: str) -> None: self.config = config self.initial_stage_name = initial_stage_name @@ -35,22 +35,32 @@ async def _do_start(self): self.current_stage = stage self.current_node_controller = NodeController() + node_program = str(stage.bin / "node") + node_arguments = stage.node_arguments + cwd = self.config.working_directory + + cwd.mkdir(parents=True, exist_ok=True) + coroutines: list[Coroutine[Any, Any, None]] = [ - # TBD: change this, start node. - self.current_node_controller.start("ls", [], Path("")), + self.current_node_controller.start(node_program, node_arguments, cwd), self.monitor_node() ] tasks = [asyncio.create_task(item) for item in coroutines] await asyncio.gather(*tasks, return_exceptions=False) + return_code = self.current_node_controller.return_code + if return_code != NODE_RETURN_CODE_SUCCESS: + break + async def monitor_node(self): assert self.current_stage is not None + assert self.current_node_controller is not None print("Node status URL:", self.current_stage.node_status_url) while True: - await asyncio.sleep(1) + await asyncio.sleep(NODE_MONITORING_PERIOD) - if self.current_node_controller and not self.current_node_controller.is_running(): + if not self.current_node_controller.is_running(): return From ff8720733a8f9fd8c79fc49aab14775bed0d58c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 4 Jul 2025 10:31:42 +0300 Subject: [PATCH 16/30] Lanes, stages, work in progress. --- multistage/lane_controller.py | 28 ++++++++----------- ...node_controller.py => stage_controller.py} | 20 +++++++++++-- 2 files changed, 29 insertions(+), 19 deletions(-) rename multistage/{node_controller.py => stage_controller.py} (69%) diff --git a/multistage/lane_controller.py b/multistage/lane_controller.py index 322c1fb..aa48657 100644 --- a/multistage/lane_controller.py +++ b/multistage/lane_controller.py @@ -4,18 +4,17 @@ from rich import print from rich.rule import Rule -from multistage.config import LaneConfig, StageConfig +from multistage.config import LaneConfig from multistage.constants import (NODE_MONITORING_PERIOD, NODE_RETURN_CODE_SUCCESS) -from multistage.node_controller import NodeController +from multistage.stage_controller import StageController class LaneController: def __init__(self, config: LaneConfig, initial_stage_name: str) -> None: self.config = config self.initial_stage_name = initial_stage_name - self.current_stage: Optional[StageConfig] = None - self.current_node_controller: Optional[NodeController] = None + self.current_stage_controller: Optional[StageController] = None def start(self): try: @@ -32,35 +31,32 @@ async def _do_start(self): for stage in stages: print(Rule(f"[bold yellow]{stage.name}")) - self.current_stage = stage - self.current_node_controller = NodeController() + self.current_stage_controller = StageController(stage) - node_program = str(stage.bin / "node") - node_arguments = stage.node_arguments cwd = self.config.working_directory - cwd.mkdir(parents=True, exist_ok=True) coroutines: list[Coroutine[Any, Any, None]] = [ - self.current_node_controller.start(node_program, node_arguments, cwd), + self.current_stage_controller.start(cwd), self.monitor_node() ] tasks = [asyncio.create_task(item) for item in coroutines] await asyncio.gather(*tasks, return_exceptions=False) - return_code = self.current_node_controller.return_code + return_code = self.current_stage_controller.return_code if return_code != NODE_RETURN_CODE_SUCCESS: break async def monitor_node(self): - assert self.current_stage is not None - assert self.current_node_controller is not None - - print("Node status URL:", self.current_stage.node_status_url) + assert self.current_stage_controller is not None while True: await asyncio.sleep(NODE_MONITORING_PERIOD) - if not self.current_node_controller.is_running(): + if not self.current_stage_controller.is_running(): return + + epoch = self.current_stage_controller.get_current_epoch() + + print("monitor_node()", epoch) diff --git a/multistage/node_controller.py b/multistage/stage_controller.py similarity index 69% rename from multistage/node_controller.py rename to multistage/stage_controller.py index 42a9dad..55186b5 100644 --- a/multistage/node_controller.py +++ b/multistage/stage_controller.py @@ -6,15 +6,26 @@ from rich import print +from multistage.config import StageConfig from multistage.constants import NODE_PROCESS_ULIMIT -class NodeController: - def __init__(self) -> None: +class StageController: + def __init__(self, config: StageConfig) -> None: + self.config = config self.process: Optional[Process] = None self.return_code = -1 - async def start(self, program: str, args: list[str], cwd: Path): + def configure(self): + # configurationArchive + # "withDbLookupExtensions": true, + # "withIndexing": false + pass + + async def start(self, cwd: Path): + program = self.config.bin / "node" + args = self.config.node_arguments + print(f"Starting node ...") print("args:", args) print("cwd:", cwd) @@ -45,3 +56,6 @@ def stop(self): def is_running(self) -> bool: return self.process is not None + + def get_current_epoch(self) -> int: + return 42 From 7b5113132949fbb05554fa9307e290ea9a2a09c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 4 Jul 2025 11:54:15 +0300 Subject: [PATCH 17/30] Configure node, work in progress. --- multistage/constants.py | 1 + multistage/lane_controller.py | 8 +++++--- multistage/shared.py | 28 ++++++++++++++++++++++++++++ multistage/stage_controller.py | 23 ++++++++++++++--------- 4 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 multistage/shared.py diff --git a/multistage/constants.py b/multistage/constants.py index f990635..d372e85 100644 --- a/multistage/constants.py +++ b/multistage/constants.py @@ -4,6 +4,7 @@ NODE_PROCESS_ULIMIT = 1024 * 512 NODE_MONITORING_PERIOD = 1 NODE_RETURN_CODE_SUCCESS = 0 +TEMPORARY_DIRECTORIES_PREFIX = "mx_chain_scripts_multistage_" # Read, write and execute by owner, read and execute by group and others FILE_MODE_NICE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH diff --git a/multistage/lane_controller.py b/multistage/lane_controller.py index aa48657..94e2fda 100644 --- a/multistage/lane_controller.py +++ b/multistage/lane_controller.py @@ -33,11 +33,13 @@ async def _do_start(self): self.current_stage_controller = StageController(stage) - cwd = self.config.working_directory - cwd.mkdir(parents=True, exist_ok=True) + working_directory = self.config.working_directory + working_directory.mkdir(parents=True, exist_ok=True) + + self.current_stage_controller.configure(working_directory) coroutines: list[Coroutine[Any, Any, None]] = [ - self.current_stage_controller.start(cwd), + self.current_stage_controller.start(working_directory), self.monitor_node() ] diff --git a/multistage/shared.py b/multistage/shared.py new file mode 100644 index 0000000..938cbdd --- /dev/null +++ b/multistage/shared.py @@ -0,0 +1,28 @@ +import shutil +import tempfile +import urllib.request +from pathlib import Path + +from rich import print + +from multistage.constants import TEMPORARY_DIRECTORIES_PREFIX + + +def fetch_archive(archive_url: str, destination_path: Path): + with tempfile.TemporaryDirectory(prefix=TEMPORARY_DIRECTORIES_PREFIX) as tmpdirname: + archive_extension = archive_url.split(".")[-1] + download_path = Path(tmpdirname) / f"archive.{archive_extension}" + extraction_path = Path(tmpdirname) / "extracted" + + print(f"Downloading archive {archive_url} to {download_path} ...") + urllib.request.urlretrieve(archive_url, download_path) + + print(f"Unpacking archive {download_path} to {extraction_path} ...") + shutil.unpack_archive(download_path, extraction_path) + + items = list(extraction_path.glob("*")) + assert len(items) == 1, "archive should have contained only one top-level item" + top_level_item = items[0] + + print(f"Moving {top_level_item} to {destination_path} ...") + shutil.move(top_level_item, destination_path) diff --git a/multistage/stage_controller.py b/multistage/stage_controller.py index 55186b5..80b0d20 100644 --- a/multistage/stage_controller.py +++ b/multistage/stage_controller.py @@ -1,5 +1,6 @@ import asyncio import os +import shutil from asyncio.subprocess import Process from pathlib import Path from typing import Optional @@ -8,6 +9,7 @@ from multistage.config import StageConfig from multistage.constants import NODE_PROCESS_ULIMIT +from multistage.shared import fetch_archive class StageController: @@ -16,33 +18,36 @@ def __init__(self, config: StageConfig) -> None: self.process: Optional[Process] = None self.return_code = -1 - def configure(self): - # configurationArchive + def configure(self, working_directory: Path): + config_directory = working_directory / "config" + shutil.rmtree(config_directory, ignore_errors=True) + fetch_archive(self.config.configuration_archive, config_directory) + # "withDbLookupExtensions": true, # "withIndexing": false pass - async def start(self, cwd: Path): + async def start(self, working_directory: Path): program = self.config.bin / "node" args = self.config.node_arguments - print(f"Starting node ...") - print("args:", args) - print("cwd:", cwd) + print(f"Starting node in {working_directory} ...") + print(args) env = os.environ.copy() - env["LD_LIBRARY_PATH"] = str(cwd) + env["LD_LIBRARY_PATH"] = str(working_directory) self.process = await asyncio.create_subprocess_exec( program, *args, - cwd=cwd, + stdout=asyncio.subprocess.DEVNULL, + cwd=working_directory, limit=NODE_PROCESS_ULIMIT, env=env, ) return_code = await self.process.wait() - print(f"Proces [{self.process.pid}] stopped. Return code: {return_code}.") + print(f"Node [{self.process.pid}] stopped, with return code = {return_code}. See node's logs.") self.process = None self.return_code = return_code From 618c8a543c23d2c8ca0501149df02f5a45c9b57b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 4 Jul 2025 13:08:26 +0300 Subject: [PATCH 18/30] Refactor download steps. --- multistage/build.py | 21 +++------------------ multistage/driver.testnet.json | 6 +++--- multistage/golang.py | 18 ++++-------------- multistage/shared.py | 7 +++++-- 4 files changed, 15 insertions(+), 37 deletions(-) diff --git a/multistage/build.py b/multistage/build.py index d5780e5..94b2004 100644 --- a/multistage/build.py +++ b/multistage/build.py @@ -3,7 +3,6 @@ import shutil import sys import traceback -import urllib.request from argparse import ArgumentParser from pathlib import Path from typing import Any @@ -15,6 +14,7 @@ from multistage import errors, golang from multistage.config import BuildConfigEntry from multistage.constants import FILE_MODE_NICE +from multistage.shared import fetch_archive def main(cli_args: list[str] = sys.argv[1:]): @@ -51,25 +51,10 @@ def _do_main(cli_args: list[str]): def do_download(workspace: Path, entry: BuildConfigEntry) -> Path: - download_folder = workspace / entry.name - extraction_folder = workspace / entry.name url = entry.source_url + extraction_folder = workspace / entry.name - shutil.rmtree(download_folder, ignore_errors=True) - download_folder.mkdir(parents=True, exist_ok=True) - - shutil.rmtree(extraction_folder, ignore_errors=True) - extraction_folder.mkdir(parents=True, exist_ok=True) - - archive_extension = url.split(".")[-1] - download_path = download_folder / f"source.{archive_extension}" - - print(f"Downloading archive {url} to {download_path}") - urllib.request.urlretrieve(url, download_path) - - print(f"Unpacking archive {download_path} to {extraction_folder}") - shutil.unpack_archive(download_path, extraction_folder, format="zip") - + fetch_archive(url, extraction_folder) return extraction_folder diff --git a/multistage/driver.testnet.json b/multistage/driver.testnet.json index adee25c..db014ef 100644 --- a/multistage/driver.testnet.json +++ b/multistage/driver.testnet.json @@ -8,7 +8,7 @@ "name": "spica", "untilEpoch": 201, "nodeStatusUrl": "http://localhost:8080/node/status", - "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/releases/tag/T1.8.12.1.zip", + "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.8.12.0.zip", "bin": "~/mvx-binaries/spica", "nodeArguments": [ "--log-save", @@ -23,7 +23,7 @@ "name": "andromeda", "untilEpoch": 767, "nodeStatusUrl": "http://localhost:8080/node/status", - "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/releases/tag/T1.9.6.0.zip", + "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.9.6.0.zip", "bin": "~/mvx-binaries/andromeda", "nodeArguments": [ "--log-save", @@ -38,7 +38,7 @@ "name": "barnard", "untilEpoch": 4294967295, "nodeStatusUrl": "http://localhost:8080/node/status", - "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/releases/tag/T1.10.1.0.zip", + "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.10.1.0.zip", "bin": "~/mvx-binaries/barnard", "nodeArguments": [ "--log-save", diff --git a/multistage/golang.py b/multistage/golang.py index 613de86..f5effbd 100644 --- a/multistage/golang.py +++ b/multistage/golang.py @@ -1,13 +1,11 @@ import os -import shutil import subprocess -import urllib.request from pathlib import Path -from urllib.parse import urlparse from rich import print from multistage import errors +from multistage.shared import fetch_archive class BuildEnvironment: @@ -31,10 +29,10 @@ def acquire_environment(workspace: Path, label: str) -> BuildEnvironment: current_system_path = os.environ.get("PATH", "") return BuildEnvironment( - system_path=f"{directory / 'go' / 'bin'}:{current_system_path}", + system_path=f"{directory / 'bin'}:{current_system_path}", go_path=str(directory / "gopath"), go_cache=str(directory / "gocache"), - go_root=str(directory / "go"), + go_root=str(directory), ) @@ -43,21 +41,13 @@ def get_environment_directory(workspace: Path, label: str) -> Path: def install_go(workspace: Path, download_url: str, environment_label: str): - download_url_parsed = urlparse(download_url) - download_url_path = download_url_parsed.path - download_to_path = workspace / Path(download_url_path).name environment_directory = get_environment_directory(workspace, environment_label) if environment_directory.exists(): print(f"Go already installed in {environment_directory} ({environment_label}).") return - print(f"Downloading {download_url} to {download_to_path} ...") - urllib.request.urlretrieve(download_url, download_to_path) - - print(f"Extracting {environment_directory} ...") - shutil.rmtree(environment_directory, ignore_errors=True) - shutil.unpack_archive(download_to_path, environment_directory) + fetch_archive(download_url, environment_directory) print(f"Creating go environment directories ...") (environment_directory / "gopath").mkdir() diff --git a/multistage/shared.py b/multistage/shared.py index 938cbdd..eee1ca6 100644 --- a/multistage/shared.py +++ b/multistage/shared.py @@ -2,6 +2,7 @@ import tempfile import urllib.request from pathlib import Path +from urllib.parse import urlparse from rich import print @@ -9,9 +10,11 @@ def fetch_archive(archive_url: str, destination_path: Path): + archive_url_parsed = urlparse(archive_url) + file_name = Path(archive_url_parsed.path).name + with tempfile.TemporaryDirectory(prefix=TEMPORARY_DIRECTORIES_PREFIX) as tmpdirname: - archive_extension = archive_url.split(".")[-1] - download_path = Path(tmpdirname) / f"archive.{archive_extension}" + download_path = Path(tmpdirname) / file_name extraction_path = Path(tmpdirname) / "extracted" print(f"Downloading archive {archive_url} to {download_path} ...") From 7f9b1639a74e2be465ee8cd731fb34536937c6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 4 Jul 2025 13:32:59 +0300 Subject: [PATCH 19/30] Sketch get_current_epoch(). --- multistage/stage_controller.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/multistage/stage_controller.py b/multistage/stage_controller.py index 80b0d20..d04aca4 100644 --- a/multistage/stage_controller.py +++ b/multistage/stage_controller.py @@ -2,9 +2,11 @@ import os import shutil from asyncio.subprocess import Process +from http import HTTPStatus from pathlib import Path from typing import Optional +import requests from rich import print from multistage.config import StageConfig @@ -63,4 +65,14 @@ def is_running(self) -> bool: return self.process is not None def get_current_epoch(self) -> int: - return 42 + status_url = self.config.node_status_url + response = requests.get(status_url) + + if response.status_code != HTTPStatus.OK: + return 0 + + data = response.json().get("data", {}) + metrics = data.get("metrics", {}) + epoch = int(metrics.get("erd_epoch_number", 0)) + + return epoch From a33d482580ca034a077ac77d3bf12e87d451b6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 4 Jul 2025 14:22:54 +0300 Subject: [PATCH 20/30] Handle prefs file etc. --- multistage/config.py | 6 ++++++ multistage/constants.py | 2 +- multistage/driver.testnet.json | 14 +++++--------- multistage/stage_controller.py | 24 +++++++++++++----------- 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/multistage/config.py b/multistage/config.py index 14f99ed..08d65fb 100644 --- a/multistage/config.py +++ b/multistage/config.py @@ -111,6 +111,7 @@ def __init__(self, until_epoch: int, node_status_url: str, configuration_archive: str, + prefs: str, bin: str, node_arguments: list[str], with_db_lookup_extensions: bool, @@ -123,6 +124,8 @@ def __init__(self, raise errors.BadConfigurationError(f"for stage {name}, 'node status url' is required") if not configuration_archive: raise errors.BadConfigurationError(f"for stage {name}, 'configuration archive' is required") + if not prefs: + raise errors.BadConfigurationError(f"for stage {name}, 'prefs' is required") if not bin: raise errors.BadConfigurationError(f"for stage {name}, 'bin' is required") @@ -130,6 +133,7 @@ def __init__(self, self.until_epoch = until_epoch self.node_status_url = node_status_url self.configuration_archive = configuration_archive + self.prefs = Path(prefs).expanduser().resolve() self.bin = Path(bin).expanduser().resolve() self.node_arguments = node_arguments self.with_db_lookup_extensions = with_db_lookup_extensions @@ -141,6 +145,7 @@ def new_from_dictionary(cls, data: dict[str, Any]): until_epoch = data.get("untilEpoch") or 0 node_status_url = data.get("nodeStatusUrl") or "" configuration_archive = data.get("configurationArchive") or "" + prefs = data.get("prefs") or "" bin = data.get("bin") or "" node_arguments = data.get("nodeArguments") or [] with_db_lookup_extensions = data.get("withDbLookupExtensions") or False @@ -151,6 +156,7 @@ def new_from_dictionary(cls, data: dict[str, Any]): until_epoch=until_epoch, node_status_url=node_status_url, configuration_archive=configuration_archive, + prefs=prefs, bin=bin, node_arguments=node_arguments, with_db_lookup_extensions=with_db_lookup_extensions, diff --git a/multistage/constants.py b/multistage/constants.py index d372e85..723aa0a 100644 --- a/multistage/constants.py +++ b/multistage/constants.py @@ -2,7 +2,7 @@ METACHAIN_ID = 4294967295 NODE_PROCESS_ULIMIT = 1024 * 512 -NODE_MONITORING_PERIOD = 1 +NODE_MONITORING_PERIOD = 5 NODE_RETURN_CODE_SUCCESS = 0 TEMPORARY_DIRECTORIES_PREFIX = "mx_chain_scripts_multistage_" diff --git a/multistage/driver.testnet.json b/multistage/driver.testnet.json index db014ef..76d67ac 100644 --- a/multistage/driver.testnet.json +++ b/multistage/driver.testnet.json @@ -9,26 +9,24 @@ "untilEpoch": 201, "nodeStatusUrl": "http://localhost:8080/node/status", "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.8.12.0.zip", + "prefs": "~/mvx-workspace/_prefs_/spica_0.toml", "bin": "~/mvx-binaries/spica", "nodeArguments": [ "--log-save", "--log-level=*:DEBUG", - "--destination-shard-as-observer=0", "--rest-api-interface=8080" - ], - "withDbLookupExtensions": true, - "withIndexing": false + ] }, { "name": "andromeda", "untilEpoch": 767, "nodeStatusUrl": "http://localhost:8080/node/status", "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.9.6.0.zip", + "prefs": "~/mvx-workspace/_prefs_/andromeda_0.toml", "bin": "~/mvx-binaries/andromeda", "nodeArguments": [ "--log-save", "--log-level=*:DEBUG", - "--destination-shard-as-observer=0", "--rest-api-interface=8080" ], "withDbLookupExtensions": true, @@ -39,15 +37,13 @@ "untilEpoch": 4294967295, "nodeStatusUrl": "http://localhost:8080/node/status", "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.10.1.0.zip", + "prefs": "~/mvx-workspace/_prefs_/barnard_0.toml", "bin": "~/mvx-binaries/barnard", "nodeArguments": [ "--log-save", "--log-level=*:DEBUG", - "--destination-shard-as-observer=0", "--rest-api-interface=8080" - ], - "withDbLookupExtensions": true, - "withIndexing": false + ] } ] } diff --git a/multistage/stage_controller.py b/multistage/stage_controller.py index d04aca4..d5ef46b 100644 --- a/multistage/stage_controller.py +++ b/multistage/stage_controller.py @@ -24,10 +24,7 @@ def configure(self, working_directory: Path): config_directory = working_directory / "config" shutil.rmtree(config_directory, ignore_errors=True) fetch_archive(self.config.configuration_archive, config_directory) - - # "withDbLookupExtensions": true, - # "withIndexing": false - pass + shutil.copy(self.config.prefs, config_directory / "prefs.toml") async def start(self, working_directory: Path): program = self.config.bin / "node" @@ -66,13 +63,18 @@ def is_running(self) -> bool: def get_current_epoch(self) -> int: status_url = self.config.node_status_url - response = requests.get(status_url) - if response.status_code != HTTPStatus.OK: - return 0 + try: + response = requests.get(status_url) - data = response.json().get("data", {}) - metrics = data.get("metrics", {}) - epoch = int(metrics.get("erd_epoch_number", 0)) + if response.status_code != HTTPStatus.OK: + return 0 - return epoch + data = response.json().get("data", {}) + metrics = data.get("metrics", {}) + epoch = int(metrics.get("erd_epoch_number", 0)) + + return epoch + except Exception as error: + print(f"[red]Error:[/red] {error}") + return 0 From 5fcd39d1c48ebacc134d21b153629d64e83b01b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 4 Jul 2025 14:25:35 +0300 Subject: [PATCH 21/30] Remove spica (and older). --- multistage/build.json | 18 ------------------ multistage/driver.testnet.json | 13 ------------- 2 files changed, 31 deletions(-) diff --git a/multistage/build.json b/multistage/build.json index 1515332..70fa351 100644 --- a/multistage/build.json +++ b/multistage/build.json @@ -1,22 +1,4 @@ [ - { - "name": "sirius", - "goUrl": "https://golang.org/dl/go1.20.7.linux-amd64.tar.gz", - "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.6.18.zip", - "destinationFolder": "~/mvx-binaries/sirius" - }, - { - "name": "vega", - "goUrl": "https://golang.org/dl/go1.20.7.linux-amd64.tar.gz", - "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.7.13-patch2.zip", - "destinationFolder": "~/mvx-binaries/vega" - }, - { - "name": "spica", - "goUrl": "https://golang.org/dl/go1.20.7.linux-amd64.tar.gz", - "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.8.12.zip", - "destinationFolder": "~/mvx-binaries/spica" - }, { "name": "andromeda", "goUrl": "https://golang.org/dl/go1.20.7.linux-amd64.tar.gz", diff --git a/multistage/driver.testnet.json b/multistage/driver.testnet.json index 76d67ac..5041d0b 100644 --- a/multistage/driver.testnet.json +++ b/multistage/driver.testnet.json @@ -4,19 +4,6 @@ "name": "shard_0", "workingDirectory": "~/mvx-nodes/shard-0", "stages": [ - { - "name": "spica", - "untilEpoch": 201, - "nodeStatusUrl": "http://localhost:8080/node/status", - "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.8.12.0.zip", - "prefs": "~/mvx-workspace/_prefs_/spica_0.toml", - "bin": "~/mvx-binaries/spica", - "nodeArguments": [ - "--log-save", - "--log-level=*:DEBUG", - "--rest-api-interface=8080" - ] - }, { "name": "andromeda", "untilEpoch": 767, From 3266ae5e9db024ecd0fd452715b20d32655bc15c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 4 Jul 2025 14:28:34 +0300 Subject: [PATCH 22/30] Fix config etc. --- multistage/README.md | 2 +- multistage/driver.testnet.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/multistage/README.md b/multistage/README.md index 25829e6..eeabae6 100644 --- a/multistage/README.md +++ b/multistage/README.md @@ -25,5 +25,5 @@ PYTHONPATH=. python3 ./multistage/build.py --workspace=~/mvx-workspace --config= ## Set up an observer (or a squad) ``` -PYTHONPATH=. python3 ./multistage/driver.py --config=./multistage/driver.testnet.json --lane=shard_0 --stage=spica +PYTHONPATH=. python3 ./multistage/driver.py --config=./multistage/driver.testnet.json --lane=shard_0 --stage=andromeda ``` diff --git a/multistage/driver.testnet.json b/multistage/driver.testnet.json index 5041d0b..67c2583 100644 --- a/multistage/driver.testnet.json +++ b/multistage/driver.testnet.json @@ -14,7 +14,7 @@ "nodeArguments": [ "--log-save", "--log-level=*:DEBUG", - "--rest-api-interface=8080" + "--rest-api-interface=localhost:8080" ], "withDbLookupExtensions": true, "withIndexing": false @@ -29,7 +29,7 @@ "nodeArguments": [ "--log-save", "--log-level=*:DEBUG", - "--rest-api-interface=8080" + "--rest-api-interface=localhost:8080" ] } ] From b87c2c74f48db8eea1ce703104e87896c379838d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 4 Jul 2025 14:48:49 +0300 Subject: [PATCH 23/30] Transition to next stage. --- multistage/constants.py | 1 + multistage/driver.testnet.json | 6 ++---- multistage/lane_controller.py | 20 +++++++++++++------- multistage/stage_controller.py | 7 ++++++- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/multistage/constants.py b/multistage/constants.py index 723aa0a..cbaa603 100644 --- a/multistage/constants.py +++ b/multistage/constants.py @@ -4,6 +4,7 @@ NODE_PROCESS_ULIMIT = 1024 * 512 NODE_MONITORING_PERIOD = 5 NODE_RETURN_CODE_SUCCESS = 0 +NODE_RETURN_CODE_SIGKILL = -9 TEMPORARY_DIRECTORIES_PREFIX = "mx_chain_scripts_multistage_" # Read, write and execute by owner, read and execute by group and others diff --git a/multistage/driver.testnet.json b/multistage/driver.testnet.json index 67c2583..f93c527 100644 --- a/multistage/driver.testnet.json +++ b/multistage/driver.testnet.json @@ -6,7 +6,7 @@ "stages": [ { "name": "andromeda", - "untilEpoch": 767, + "untilEpoch": 4, "nodeStatusUrl": "http://localhost:8080/node/status", "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.9.6.0.zip", "prefs": "~/mvx-workspace/_prefs_/andromeda_0.toml", @@ -15,9 +15,7 @@ "--log-save", "--log-level=*:DEBUG", "--rest-api-interface=localhost:8080" - ], - "withDbLookupExtensions": true, - "withIndexing": false + ] }, { "name": "barnard", diff --git a/multistage/lane_controller.py b/multistage/lane_controller.py index 94e2fda..279b445 100644 --- a/multistage/lane_controller.py +++ b/multistage/lane_controller.py @@ -6,6 +6,7 @@ from multistage.config import LaneConfig from multistage.constants import (NODE_MONITORING_PERIOD, + NODE_RETURN_CODE_SIGKILL, NODE_RETURN_CODE_SUCCESS) from multistage.stage_controller import StageController @@ -40,25 +41,30 @@ async def _do_start(self): coroutines: list[Coroutine[Any, Any, None]] = [ self.current_stage_controller.start(working_directory), - self.monitor_node() + self.monitor_stage() ] tasks = [asyncio.create_task(item) for item in coroutines] await asyncio.gather(*tasks, return_exceptions=False) return_code = self.current_stage_controller.return_code + + if return_code == NODE_RETURN_CODE_SIGKILL: + continue if return_code != NODE_RETURN_CODE_SUCCESS: break - async def monitor_node(self): - assert self.current_stage_controller is not None + async def monitor_stage(self): + controller = self.current_stage_controller + + assert controller is not None while True: await asyncio.sleep(NODE_MONITORING_PERIOD) - if not self.current_stage_controller.is_running(): + if not controller.is_running(): return - epoch = self.current_stage_controller.get_current_epoch() - - print("monitor_node()", epoch) + if controller.should_stop(): + controller.stop() + return diff --git a/multistage/stage_controller.py b/multistage/stage_controller.py index d5ef46b..91d682b 100644 --- a/multistage/stage_controller.py +++ b/multistage/stage_controller.py @@ -46,7 +46,7 @@ async def start(self, working_directory: Path): ) return_code = await self.process.wait() - print(f"Node [{self.process.pid}] stopped, with return code = {return_code}. See node's logs.") + print(f"Node stopped, with return code = {return_code}. See node's logs.") self.process = None self.return_code = return_code @@ -61,6 +61,11 @@ def stop(self): def is_running(self) -> bool: return self.process is not None + def should_stop(self) -> bool: + epoch = self.get_current_epoch() + print(f"Epoch: {epoch}") + return epoch > self.config.until_epoch + def get_current_epoch(self) -> int: status_url = self.config.node_status_url From 81ca8f61989e684199e6933589c042805c7f7775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 4 Jul 2025 16:10:14 +0300 Subject: [PATCH 24/30] Prefs not needed - node args are sufficient. --- multistage/config.py | 6 ------ multistage/driver.testnet.json | 10 ++++++---- multistage/stage_controller.py | 4 +++- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/multistage/config.py b/multistage/config.py index 08d65fb..14f99ed 100644 --- a/multistage/config.py +++ b/multistage/config.py @@ -111,7 +111,6 @@ def __init__(self, until_epoch: int, node_status_url: str, configuration_archive: str, - prefs: str, bin: str, node_arguments: list[str], with_db_lookup_extensions: bool, @@ -124,8 +123,6 @@ def __init__(self, raise errors.BadConfigurationError(f"for stage {name}, 'node status url' is required") if not configuration_archive: raise errors.BadConfigurationError(f"for stage {name}, 'configuration archive' is required") - if not prefs: - raise errors.BadConfigurationError(f"for stage {name}, 'prefs' is required") if not bin: raise errors.BadConfigurationError(f"for stage {name}, 'bin' is required") @@ -133,7 +130,6 @@ def __init__(self, self.until_epoch = until_epoch self.node_status_url = node_status_url self.configuration_archive = configuration_archive - self.prefs = Path(prefs).expanduser().resolve() self.bin = Path(bin).expanduser().resolve() self.node_arguments = node_arguments self.with_db_lookup_extensions = with_db_lookup_extensions @@ -145,7 +141,6 @@ def new_from_dictionary(cls, data: dict[str, Any]): until_epoch = data.get("untilEpoch") or 0 node_status_url = data.get("nodeStatusUrl") or "" configuration_archive = data.get("configurationArchive") or "" - prefs = data.get("prefs") or "" bin = data.get("bin") or "" node_arguments = data.get("nodeArguments") or [] with_db_lookup_extensions = data.get("withDbLookupExtensions") or False @@ -156,7 +151,6 @@ def new_from_dictionary(cls, data: dict[str, Any]): until_epoch=until_epoch, node_status_url=node_status_url, configuration_archive=configuration_archive, - prefs=prefs, bin=bin, node_arguments=node_arguments, with_db_lookup_extensions=with_db_lookup_extensions, diff --git a/multistage/driver.testnet.json b/multistage/driver.testnet.json index f93c527..5095685 100644 --- a/multistage/driver.testnet.json +++ b/multistage/driver.testnet.json @@ -9,12 +9,13 @@ "untilEpoch": 4, "nodeStatusUrl": "http://localhost:8080/node/status", "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.9.6.0.zip", - "prefs": "~/mvx-workspace/_prefs_/andromeda_0.toml", "bin": "~/mvx-binaries/andromeda", "nodeArguments": [ "--log-save", "--log-level=*:DEBUG", - "--rest-api-interface=localhost:8080" + "--rest-api-interface=localhost:8080", + "--operation-mode=full-archive", + "--destination-shard-as-observer=0" ] }, { @@ -22,12 +23,13 @@ "untilEpoch": 4294967295, "nodeStatusUrl": "http://localhost:8080/node/status", "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.10.1.0.zip", - "prefs": "~/mvx-workspace/_prefs_/barnard_0.toml", "bin": "~/mvx-binaries/barnard", "nodeArguments": [ "--log-save", "--log-level=*:DEBUG", - "--rest-api-interface=localhost:8080" + "--rest-api-interface=localhost:8080", + "--operation-mode=full-archive", + "--destination-shard-as-observer=0" ] } ] diff --git a/multistage/stage_controller.py b/multistage/stage_controller.py index 91d682b..e828554 100644 --- a/multistage/stage_controller.py +++ b/multistage/stage_controller.py @@ -24,7 +24,6 @@ def configure(self, working_directory: Path): config_directory = working_directory / "config" shutil.rmtree(config_directory, ignore_errors=True) fetch_archive(self.config.configuration_archive, config_directory) - shutil.copy(self.config.prefs, config_directory / "prefs.toml") async def start(self, working_directory: Path): program = self.config.bin / "node" @@ -78,6 +77,9 @@ def get_current_epoch(self) -> int: data = response.json().get("data", {}) metrics = data.get("metrics", {}) epoch = int(metrics.get("erd_epoch_number", 0)) + nonce = int(metrics.get("erd_nonce", 0)) + + print(epoch, nonce) return epoch except Exception as error: From d05e7cf42623c90330a0deae63a8d91d73700721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 4 Jul 2025 16:12:28 +0300 Subject: [PATCH 25/30] Better logging. --- multistage/stage_controller.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/multistage/stage_controller.py b/multistage/stage_controller.py index e828554..d74c42e 100644 --- a/multistage/stage_controller.py +++ b/multistage/stage_controller.py @@ -62,7 +62,6 @@ def is_running(self) -> bool: def should_stop(self) -> bool: epoch = self.get_current_epoch() - print(f"Epoch: {epoch}") return epoch > self.config.until_epoch def get_current_epoch(self) -> int: @@ -77,9 +76,9 @@ def get_current_epoch(self) -> int: data = response.json().get("data", {}) metrics = data.get("metrics", {}) epoch = int(metrics.get("erd_epoch_number", 0)) - nonce = int(metrics.get("erd_nonce", 0)) + block_nonce = int(metrics.get("erd_nonce", 0)) - print(epoch, nonce) + print(f"Epoch = {epoch}, block = {block_nonce}") return epoch except Exception as error: From 6af0cf605ccb6aa1ce5840df7ce5338323359c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Fri, 4 Jul 2025 17:27:54 +0300 Subject: [PATCH 26/30] Partial fix after review. --- multistage/golang.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multistage/golang.py b/multistage/golang.py index f5effbd..2f7fc65 100644 --- a/multistage/golang.py +++ b/multistage/golang.py @@ -57,6 +57,6 @@ def install_go(workspace: Path, download_url: str, environment_label: str): def build(source_code: Path, environment: BuildEnvironment): print(f"Building {source_code} ...") - return_code = subprocess.check_call(["go", "build"], cwd=source_code, env=environment.to_dictionary()) + return_code = subprocess.call(["go", "build"], cwd=source_code, env=environment.to_dictionary()) if return_code != 0: raise errors.KnownError(f"error code = {return_code}, see output") From 72a64fc02b78edaaec23f1b8718940dbe9789fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 7 Jul 2025 10:01:33 +0300 Subject: [PATCH 27/30] Fix after review. --- multistage/build.py | 9 +++++++-- multistage/config.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/multistage/build.py b/multistage/build.py index 94b2004..5ef7773 100644 --- a/multistage/build.py +++ b/multistage/build.py @@ -86,8 +86,13 @@ def copy_wasmer_libraries(build_environment: golang.BuildEnvironment, go_mod: Pa def get_chain_vm_go_folder_name(go_mod: Path) -> str: lines = go_mod.read_text().splitlines() - line = [line for line in lines if "github.com/multiversx/mx-chain-vm-go" in line][0] - parts = line.split() + + matching_lines = [line for line in lines if "github.com/multiversx/mx-chain-vm-go" in line] + if not matching_lines: + raise errors.KnownError("cannot detect location of mx-chain-vm-go") + + line_of_interest = matching_lines[0] + parts = line_of_interest.split() return f"{parts[0]}@{parts[1]}" diff --git a/multistage/config.py b/multistage/config.py index 14f99ed..a5b8cfc 100644 --- a/multistage/config.py +++ b/multistage/config.py @@ -75,7 +75,7 @@ def __init__(self, name: str, working_directory: str, stages: list["StageConfig" raise errors.BadConfigurationError(f"for lane {name}, 'working directory' is required") if not stages: raise errors.BadConfigurationError(f"for lane {name}, 'stages' are required") - if len(stages) > len(set(stages)): + if len(stages) > len(set(stages_names)): raise errors.BadConfigurationError("stages names must be unique") self.name = name From c77197af52a52e86947639e499e086d19a6b8d7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 7 Jul 2025 11:25:16 +0300 Subject: [PATCH 28/30] Fix after review. --- multistage/lane_controller.py | 5 +---- multistage/shared.py | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/multistage/lane_controller.py b/multistage/lane_controller.py index 279b445..f6e788f 100644 --- a/multistage/lane_controller.py +++ b/multistage/lane_controller.py @@ -19,10 +19,7 @@ def __init__(self, config: LaneConfig, initial_stage_name: str) -> None: def start(self): try: - loop = asyncio.get_event_loop() - loop.run_until_complete(self._do_start()) - loop.close() - asyncio.set_event_loop(asyncio.new_event_loop()) + asyncio.run(self._do_start()) except KeyboardInterrupt: print("Processing lane interrupted.") diff --git a/multistage/shared.py b/multistage/shared.py index eee1ca6..5d21ec5 100644 --- a/multistage/shared.py +++ b/multistage/shared.py @@ -6,6 +6,7 @@ from rich import print +from multistage import errors from multistage.constants import TEMPORARY_DIRECTORIES_PREFIX @@ -24,7 +25,9 @@ def fetch_archive(archive_url: str, destination_path: Path): shutil.unpack_archive(download_path, extraction_path) items = list(extraction_path.glob("*")) - assert len(items) == 1, "archive should have contained only one top-level item" + if len(items) != 1: + raise errors.KnownError(f"archive {archive_url} should have contained only one top-level item") + top_level_item = items[0] print(f"Moving {top_level_item} to {destination_path} ...") From a02f48f88f0f8a6f7cfebb84249928965354cdeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 7 Jul 2025 14:57:19 +0300 Subject: [PATCH 29/30] Additional config, more explanations. --- multistage/README.md | 19 ++++- multistage/{ => samples}/build.json | 0 .../testnet_import_db.json} | 6 +- multistage/samples/testnet_sync.json | 72 +++++++++++++++++++ 4 files changed, 93 insertions(+), 4 deletions(-) rename multistage/{ => samples}/build.json (100%) rename multistage/{driver.testnet.json => samples/testnet_import_db.json} (84%) create mode 100644 multistage/samples/testnet_sync.json diff --git a/multistage/README.md b/multistage/README.md index eeabae6..83c56d7 100644 --- a/multistage/README.md +++ b/multistage/README.md @@ -19,11 +19,26 @@ pip install -r ./requirements.txt --upgrade Skip this flow if you choose to download the pre-built Node artifacts, instead of building them. ``` -PYTHONPATH=. python3 ./multistage/build.py --workspace=~/mvx-workspace --config=./multistage/build.json +PYTHONPATH=. python3 ./multistage/build.py --workspace=~/mvx-workspace --config=./multistage/samples/build.json ``` ## Set up an observer (or a squad) ``` -PYTHONPATH=. python3 ./multistage/driver.py --config=./multistage/driver.testnet.json --lane=shard_0 --stage=andromeda +PYTHONPATH=. python3 ./multistage/driver.py --config=./multistage/samples/testnet_sync.json --lane=shard_0 --stage=andromeda + +PYTHONPATH=. python3 ./multistage/driver.py --config=./multistage/samples/testnet_sync.json --lane=shard_1 --stage=andromeda +... +``` + +Once nodes are ready (synchronized to the network), switch to the regular node management scripts. + +## Run import-db + +``` +PYTHONPATH=. python3 ./multistage/driver.py --config=./multistage/samples/testnet_import_db.json --lane=shard_0 --stage=andromeda + +PYTHONPATH=. python3 ./multistage/driver.py --config=./multistage/samples/testnet_import_db.json --lane=shard_1 --stage=andromeda + +... ``` diff --git a/multistage/build.json b/multistage/samples/build.json similarity index 100% rename from multistage/build.json rename to multistage/samples/build.json diff --git a/multistage/driver.testnet.json b/multistage/samples/testnet_import_db.json similarity index 84% rename from multistage/driver.testnet.json rename to multistage/samples/testnet_import_db.json index 5095685..df97cfb 100644 --- a/multistage/driver.testnet.json +++ b/multistage/samples/testnet_import_db.json @@ -15,7 +15,8 @@ "--log-level=*:DEBUG", "--rest-api-interface=localhost:8080", "--operation-mode=full-archive", - "--destination-shard-as-observer=0" + "--destination-shard-as-observer=0", + "--import-db=~/mvx-nodes/shard-0/import-db" ] }, { @@ -29,7 +30,8 @@ "--log-level=*:DEBUG", "--rest-api-interface=localhost:8080", "--operation-mode=full-archive", - "--destination-shard-as-observer=0" + "--destination-shard-as-observer=0", + "--import-db=~/mvx-nodes/shard-0/import-db" ] } ] diff --git a/multistage/samples/testnet_sync.json b/multistage/samples/testnet_sync.json new file mode 100644 index 0000000..03a89d1 --- /dev/null +++ b/multistage/samples/testnet_sync.json @@ -0,0 +1,72 @@ +{ + "lanes": [ + { + "name": "shard_0", + "workingDirectory": "~/mvx-nodes/shard-0", + "stages": [ + { + "name": "andromeda", + "untilEpoch": 4, + "nodeStatusUrl": "http://localhost:8080/node/status", + "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.9.6.0.zip", + "bin": "~/mvx-binaries/andromeda", + "nodeArguments": [ + "--log-save", + "--log-level=*:DEBUG", + "--rest-api-interface=localhost:8080", + "--operation-mode=full-archive", + "--destination-shard-as-observer=0" + ] + }, + { + "name": "barnard", + "untilEpoch": 4294967295, + "nodeStatusUrl": "http://localhost:8080/node/status", + "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.10.1.0.zip", + "bin": "~/mvx-binaries/barnard", + "nodeArguments": [ + "--log-save", + "--log-level=*:DEBUG", + "--rest-api-interface=localhost:8080", + "--operation-mode=full-archive", + "--destination-shard-as-observer=0" + ] + } + ] + }, + { + "name": "shard_1", + "workingDirectory": "~/mvx-nodes/shard-1", + "stages": [ + { + "name": "andromeda", + "untilEpoch": 4, + "nodeStatusUrl": "http://localhost:8081/node/status", + "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.9.6.0.zip", + "bin": "~/mvx-binaries/andromeda", + "nodeArguments": [ + "--log-save", + "--log-level=*:DEBUG", + "--rest-api-interface=localhost:8081", + "--operation-mode=full-archive", + "--destination-shard-as-observer=1" + ] + }, + { + "name": "barnard", + "untilEpoch": 4294967295, + "nodeStatusUrl": "http://localhost:8081/node/status", + "configurationArchive": "https://github.com/multiversx/mx-chain-testnet-config/archive/refs/tags/T1.10.1.0.zip", + "bin": "~/mvx-binaries/barnard", + "nodeArguments": [ + "--log-save", + "--log-level=*:DEBUG", + "--rest-api-interface=localhost:8081", + "--operation-mode=full-archive", + "--destination-shard-as-observer=1" + ] + } + ] + } + ] +} From c8394b9cc73fb6325308718e60d2eec83daaf570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Mon, 7 Jul 2025 14:57:36 +0300 Subject: [PATCH 30/30] Handle ~ in args. --- multistage/stage_controller.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/multistage/stage_controller.py b/multistage/stage_controller.py index d74c42e..92be641 100644 --- a/multistage/stage_controller.py +++ b/multistage/stage_controller.py @@ -29,6 +29,9 @@ async def start(self, working_directory: Path): program = self.config.bin / "node" args = self.config.node_arguments + # Handle "~" in args: + args = [arg.replace("~", str(Path.home())) for arg in args] + print(f"Starting node in {working_directory} ...") print(args)