diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..e44b810 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +ignore = E501 diff --git a/.gitignore b/.gitignore index 1563977..0eaa8a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ #Don't track files: .DS_Store +venv/ +__pycache__ diff --git a/multiversion/README.md b/multiversion/README.md new file mode 100644 index 0000000..3d3d6f7 --- /dev/null +++ b/multiversion/README.md @@ -0,0 +1,27 @@ +# MultiversX multi-version node scripts + +These scripts allow one to use multiple versions of the MultiversX node, in sequence, to _sync_ (from the deep past) or run _import-db_ flows. + +**Important:** these scripts are only suitable for observers, not for validators. Furthermore, the MultiversX proxy isn't handled. + +## Building the artifacts + +Skip this flow if you choose to download the pre-built Node artifacts, instead of building them. + +Go must be installed beforehand. + +``` +PYTHONPATH=. python3 ./multiversion/build.py --workspace=~/mvx-workspace --config=./multiversion/build.json +``` + +## Maintenance + +### Python virtual environment + +Create a virtual environment and install the dependencies: + +``` +python3 -m venv ./venv +source ./venv/bin/activate +pip install -r ./requirements.txt --upgrade +``` diff --git a/multiversion/build.json b/multiversion/build.json new file mode 100644 index 0000000..fa521b2 --- /dev/null +++ b/multiversion/build.json @@ -0,0 +1,22 @@ +[ + { + "name": "sirius", + "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.6.18.zip", + "destinationFolder": "~/mvx-binaries/sirius" + }, + { + "name": "vega", + "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.7.13-patch2.zip", + "destinationFolder": "~/mvx-binaries/vega" + }, + { + "name": "spica", + "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.8.12.zip", + "destinationFolder": "~/mvx-binaries/spica" + }, + { + "name": "andromeda", + "sourceUrl": "https://github.com/multiversx/mx-chain-go/archive/refs/tags/v1.9.0.zip", + "destinationFolder": "~/mvx-binaries/andromeda" + } +] diff --git a/multiversion/build.py b/multiversion/build.py new file mode 100644 index 0000000..5a01b43 --- /dev/null +++ b/multiversion/build.py @@ -0,0 +1,154 @@ +import json +import os +import shutil +import subprocess +import sys +import traceback +import urllib.request +from argparse import ArgumentParser +from pathlib import Path +from typing import Any + +from rich import print +from rich.panel import Panel +from rich.rule import Rule + +from multiversion import errors +from multiversion.constants import FILE_MODE_NICE + + +class BuildConfigEntry: + def __init__(self, name: str, source_url: str, destination_folder: str) -> None: + if not name: + raise errors.KnownError("build 'name' is required") + if not source_url: + raise errors.KnownError("build 'source' is required") + if not destination_folder: + raise errors.KnownError("build 'destination' is required") + + self.name = name + self.source_url = source_url + self.destination_folder = destination_folder + + @classmethod + def new_from_dictionary(cls, data: dict[str, Any]): + name = data.get("name") or "" + source_url = data.get("sourceUrl") or "" + destination_folder = data.get("destinationFolder") or "" + + return cls( + name=name, + source_url=source_url, + destination_folder=destination_folder, + ) + + +def main(cli_args: list[str] = sys.argv[1:]): + try: + _do_main(cli_args) + except errors.KnownError as err: + print(Panel(f"[red]{traceback.format_exc()}")) + print(Panel(f"[red]{err.get_pretty()}")) + return 1 + + +def _do_main(cli_args: list[str]): + parser = ArgumentParser() + parser.add_argument("--workspace", required=True, help="path of the build workspace") + parser.add_argument("--config", required=True, help="path of the 'build' configuration file") + args = parser.parse_args(cli_args) + + workspace_path = Path(args.workspace).expanduser().resolve() + workspace_path.mkdir(parents=True, exist_ok=True) + + config_path = Path(args.config).expanduser().resolve() + config_data = json.loads(config_path.read_text()) + config_entries = [BuildConfigEntry.new_from_dictionary(item) for item in config_data] + + for entry in config_entries: + print(Rule(f"[bold yellow]{entry.name}")) + source_parent_folder = do_download(workspace_path, entry) + cmd_node_folder = do_build(source_parent_folder) + copy_artifacts(cmd_node_folder, entry) + + +def do_download(workspace: Path, entry: BuildConfigEntry) -> Path: + download_folder = workspace / entry.name + extraction_folder = workspace / entry.name + url = entry.source_url + + print(f"Re-creating {download_folder} ...") + shutil.rmtree(download_folder, ignore_errors=True) + download_folder.mkdir(parents=True, exist_ok=True) + + print(f"Re-creating {extraction_folder} ...") + shutil.rmtree(extraction_folder, ignore_errors=True) + extraction_folder.mkdir(parents=True, exist_ok=True) + + archive_extension = url.split(".")[-1] + download_path = download_folder / f"source.{archive_extension}" + + print(f"Downloading archive {url} to {download_path}") + urllib.request.urlretrieve(url, download_path) + + print(f"Unpacking archive {download_path} to {extraction_folder}") + shutil.unpack_archive(download_path, extraction_folder, format="zip") + + return extraction_folder + + +def do_build(source_parent_folder: Path) -> Path: + # If has one subfolder, that one is the source code + subfolders = [Path(item.path) for item in os.scandir(source_parent_folder) if item.is_dir()] + source_folder = subfolders[0] if len(subfolders) == 1 else source_parent_folder + + cmd_node = source_folder / "cmd" / "node" + go_mod = source_folder / "go.mod" + + print(f"Building {cmd_node} ...") + + return_code = subprocess.check_call(["go", "build"], cwd=cmd_node) + if return_code != 0: + raise errors.KnownError(f"error code = {return_code}, see output") + + copy_wasmer_libraries(go_mod, cmd_node) + return cmd_node + + +def copy_wasmer_libraries(go_mod: Path, destination: Path): + go_path_variable = os.environ.get("GOPATH", "~/go") + go_path = Path(go_path_variable).expanduser().resolve() + vm_go_folder_name = get_chain_vm_go_folder_name(go_mod) + vm_go_path = go_path / "pkg" / "mod" / vm_go_folder_name + libraries = list((vm_go_path / "wasmer").glob("*.so")) + list((vm_go_path / "wasmer2").glob("*.so")) + + for library in libraries: + shutil.copy(library, destination) + os.chmod(destination / library.name, FILE_MODE_NICE) + + +def get_chain_vm_go_folder_name(go_mod: Path) -> str: + lines = go_mod.read_text().splitlines() + line = [line for line in lines if "github.com/multiversx/mx-chain-vm-go" in line][0] + parts = line.split() + return f"{parts[0]}@{parts[1]}" + + +def copy_artifacts(cmd_node_folder: Path, entry: BuildConfigEntry): + print(f"Copying artifacts to {entry.destination_folder} ...") + + libraries = list(cmd_node_folder.glob("*.so")) + executable = cmd_node_folder / "node" + artifacts = libraries + [executable] + + destination_folder = Path(entry.destination_folder).expanduser().resolve() + shutil.rmtree(destination_folder, ignore_errors=True) + destination_folder.mkdir(parents=True, exist_ok=True) + + for artifact in artifacts: + shutil.copy(artifact, destination_folder) + + +if __name__ == "__main__": + ret = main(sys.argv[1:]) + sys.exit(ret) diff --git a/multiversion/constants.py b/multiversion/constants.py new file mode 100644 index 0000000..a791d12 --- /dev/null +++ b/multiversion/constants.py @@ -0,0 +1,6 @@ +import stat + +METACHAIN_ID = 4294967295 + +# Read, write and execute by owner, read and execute by group and others +FILE_MODE_NICE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH diff --git a/multiversion/driver.py b/multiversion/driver.py new file mode 100644 index 0000000..f16b99c --- /dev/null +++ b/multiversion/driver.py @@ -0,0 +1,28 @@ +import sys +import traceback +from argparse import ArgumentParser + +from rich import print +from rich.panel import Panel + +from multiversion import errors + + +def main(cli_args: list[str] = sys.argv[1:]): + try: + _do_main(cli_args) + except errors.KnownError as err: + print(Panel(f"[red]{traceback.format_exc()}")) + print(Panel(f"[red]{err.get_pretty()}")) + return 1 + + +def _do_main(cli_args: list[str]): + parser = ArgumentParser() + args = parser.parse_args(cli_args) + + +if __name__ == "__main__": + ret = main(sys.argv[1:]) + sys.exit(ret) + diff --git a/multiversion/errors.py b/multiversion/errors.py new file mode 100644 index 0000000..5b84e62 --- /dev/null +++ b/multiversion/errors.py @@ -0,0 +1,29 @@ +from typing import Any, Optional + + +class KnownError(Exception): + def __init__(self, message: str, inner: Optional[Any] = None): + super().__init__(message) + self.inner = inner + + def get_pretty(self) -> str: + if self.inner: + return f"""{self} +... {self.inner} +""" + return str(self) + + +class BadConfigurationError(KnownError): + def __init__(self, message: str): + super().__init__(f"bad configuration: {message}") + + +class UsageError(KnownError): + def __init__(self, message: str): + super().__init__(f"bad usage: {message}") + + +class TransientError(KnownError): + def __init__(self, message: str, inner: Optional[Any] = None): + super().__init__(f"transient error: {message}", inner) diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..72a3c50 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,14 @@ +{ + "include": ["multiversion"], + "exclude": ["**/__pycache__"], + "ignore": [], + "defineConstant": { + "DEBUG": true + }, + "venvPath": ".", + "venv": "venv", + "stubPath": "", + "reportMissingImports": true, + "reportMissingTypeStubs": false, + "reportUnknownParameterType": true +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9b0bc41 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests>=2.32.0,<3.0.0 +rich==13.9.4