From 162d4ceab7ff38a54c6b058118f3b1d77350ff76 Mon Sep 17 00:00:00 2001 From: h-grieve Date: Mon, 11 Aug 2025 15:07:11 +0100 Subject: [PATCH 01/11] add race condition handler --- src/blockassist/data.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/blockassist/data.py b/src/blockassist/data.py index a2bb59f..33c7b75 100644 --- a/src/blockassist/data.py +++ b/src/blockassist/data.py @@ -1,4 +1,5 @@ import shutil +import time import zipfile from pathlib import Path @@ -99,6 +100,19 @@ def zip_and_upload_episodes( s3_uris = [] for evaluate_dir in evaluate_dirs: + # Wait for the directory to be fully created (race condition fix) + max_wait_seconds = 180 + wait_interval = 0.5 + waited = 0 + + while not evaluate_dir.exists() and waited < max_wait_seconds: + _LOG.info(f"Waiting for evaluation directory to be created: {evaluate_dir} (waited {waited:.1f}s/{max_wait_seconds}s)") + time.sleep(wait_interval) + waited += wait_interval + + if not evaluate_dir.exists(): + raise FileNotFoundError(f"Evaluation directory does not exist after waiting {max_wait_seconds}s: {evaluate_dir}") + _LOG.info(f"Processing evaluation directory: {evaluate_dir}") # Create zip file of the directory From 6c349db37187241f584b8c97739c0dcbce3a23e4 Mon Sep 17 00:00:00 2001 From: Eamonn Nugent <4699217+space55@users.noreply.github.com> Date: Fri, 5 Sep 2025 12:38:01 -0400 Subject: [PATCH 02/11] Use only one HF repo per user --- .gitignore | 1 + src/blockassist/launch.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 5bf1ef7..60d87a7 100644 --- a/.gitignore +++ b/.gitignore @@ -213,6 +213,7 @@ __marimo__/ ## Project data/base_checkpoint/evaluate* data/evaluate/ +data/evaluate_*.zip data/rllib/ episode_runs/ convert_runs/ diff --git a/src/blockassist/launch.py b/src/blockassist/launch.py index d945b2a..3ebbdaa 100644 --- a/src/blockassist/launch.py +++ b/src/blockassist/launch.py @@ -58,6 +58,7 @@ class Stage(Enum): Stage.UPLOAD_MODEL, ] + def get_stages(cfg: DictConfig) -> list[Stage]: # Overrides mode if "stages" in cfg and cfg["stages"]: @@ -78,12 +79,14 @@ def hf_login(cfg: DictConfig): def get_hf_repo_id(hf_token: str, training_id: str): username = whoami(token=hf_token)["name"] - return f"{username}/blockassist-bc-{training_id}" + return f"{username}/blockassist" async def _main(cfg: DictConfig): try: - logging.basicConfig(filename='logs/blockassist.log', encoding='utf-8', level=logging.DEBUG) + logging.basicConfig( + filename="logs/blockassist.log", encoding="utf-8", level=logging.DEBUG + ) if cfg["mode"] == "e2e": _LOG.info("Starting full recording session!!") From 247ba4bee51ab884c9e1b64f03afa268504a5ae1 Mon Sep 17 00:00:00 2001 From: Eamonn Nugent <4699217+space55@users.noreply.github.com> Date: Mon, 8 Sep 2025 09:35:46 -0400 Subject: [PATCH 03/11] Add Rich text UI --- README.md | 8 +- run.py | 323 ++++++++++++++++++++++++---------------- scripts/gradle_setup.sh | 8 +- scripts/node_env.sh | 15 +- scripts/yarn_run.sh | 4 +- scripts/yarn_setup.sh | 4 +- 6 files changed, 211 insertions(+), 151 deletions(-) diff --git a/README.md b/README.md index 014e615..c3e0578 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,10 @@ brew install pyenv pyenv install 3.10 ``` -**Step 5: Install `psutil` and `readchar`** +**Step 5: Install `psutil`, `readchar`, and `rich`** ```bash -pyenv exec pip install psutil readchar +pyenv exec pip install psutil readchar rich ``` ## Installation (Linux) @@ -90,10 +90,10 @@ sudo apt install make build-essential libssl-dev zlib1g-dev libbz2-dev libreadli pyenv install 3.10 ``` -**Step 5: Install `psutil` and `readchar`** +**Step 5: Install `psutil`, `readchar`, and `rich`** ```bash -pip install psutil readchar +pip install psutil readchar rich ``` ## Run BlockAssist diff --git a/run.py b/run.py index a440b76..b999611 100644 --- a/run.py +++ b/run.py @@ -7,6 +7,16 @@ import time from subprocess import Popen from typing import Dict, Optional +from rich.console import Console +from rich.markdown import Markdown +from rich.progress import ( + Progress, + TimeElapsedColumn, + SpinnerColumn, + TextColumn, + BarColumn, + TaskProgressColumn, +) import psutil import readchar @@ -15,18 +25,27 @@ TOTAL_TIME_PLAYED = 0 EPISODES_PLAYED = 0 +CONSOLE = Console() +LOG_COLOR = "dim" +HEADER_COLOR = "bold blue" +INFO_COLOR = "cyan" +ERROR_COLOR = "bold red" +WARNING_COLOR = "yellow" +SUCCESS_COLOR = "green" +DELINEATOR_COLOR = "bold white" +GENSYN_COLOR = "bold magenta" def create_logs_dir(clear_existing=True): if os.path.exists("logs") and clear_existing: - print("Clearing existing logs directory") + CONSOLE.print("Clearing existing logs directory", style=LOG_COLOR) cmd = "rm -rf logs" process = Popen(cmd, shell=True) ret = process.wait() if ret != 0: sys.exit(ret) - print("Creating logs directory") + CONSOLE.print("Creating logs directory", style=LOG_COLOR) cmd = "mkdir -p logs" process = Popen(cmd, shell=True) ret = process.wait() @@ -36,14 +55,14 @@ def create_logs_dir(clear_existing=True): def create_evaluate_dir(): if not os.path.exists("data/base_checkpoint/evaluate"): - print("Creating evaluate directory") + CONSOLE.print("Creating evaluate directory", style=LOG_COLOR) cmd = "mkdir -p data/base_checkpoint/evaluate" process = Popen(cmd, shell=True) ret = process.wait() if ret != 0: sys.exit(ret) else: - print("Evaluate directory already exists") + CONSOLE.print("Evaluate directory already exists", style=LOG_COLOR) def setup_venv(): @@ -122,14 +141,17 @@ def wait_for_keys(keys=_ENTER_KEYS, on_received=None): on_received(char) break else: - print( - f"Unknown key pressed: {repr(char)}. Please press a valid key in ({keys}) to continue." + CONSOLE.print( + f"Unknown key pressed: {repr(char)}. Please press a valid key in ({keys}) to continue.", + style=WARNING_COLOR, ) def send_blockassist_sigint(pid: int): logging.info("Running send_blockassist_sigint") - print("Sending SIGINT to BlockAssist process with PID:", pid) + CONSOLE.print( + f"Sending SIGINT to BlockAssist process with PID: {pid}", style=LOG_COLOR + ) parent_process = psutil.Process(pid) if parent_process.is_running(): @@ -167,12 +189,12 @@ def train_blockassist(env: Optional[Dict] = None): def wait_for_login(): logging.info("Running wait_for_login") # Extract environment variables from userData.json - print("Waiting for modal userData.json to be created...") + CONSOLE.print("Waiting for modal userData.json to be created...", style=INFO_COLOR) user_data_path = "modal-login/temp-data/userData.json" user_api_key_path = "modal-login/temp-data/userApiKey.json" while not os.path.exists(user_data_path): time.sleep(1) - print("Found userData.json. Proceeding...") + CONSOLE.print("Found userData.json. Proceeding...", style=SUCCESS_COLOR) # Read and parse the JSON file while True: @@ -186,8 +208,8 @@ def wait_for_login(): d = os.environ.copy() for k in user_data.keys(): - d["BA_ORG_ID"] = user_data[k]['orgId'] - d["BA_ADDRESS_EOA"] = user_data[k]['address'] + d["BA_ORG_ID"] = user_data[k]["orgId"] + d["BA_ADDRESS_EOA"] = user_data[k]["address"] d["PYTHONWARNINGS"] = "ignore::DeprecationWarning" for k in user_api_key.keys(): @@ -195,15 +217,14 @@ def wait_for_login(): d["BA_ADDRESS_ACCOUNT"] = user_api_key[k][-1]["accountAddress"] return d except Exception as e: - print("Waiting...") - time.sleep(1) - + CONSOLE.print("Waiting...", style=INFO_COLOR) + time.sleep(1) def run(): global TOTAL_TIME_PLAYED global EPISODES_PLAYED - print("Creating directories...") + CONSOLE.print("Creating directories...", style=LOG_COLOR) create_logs_dir(clear_existing=True) create_evaluate_dir() @@ -215,7 +236,7 @@ def run(): ) logging.info("Running BlockAssist run.py script") - print( + CONSOLE.print( """ ██████╗ ██╗ ██████╗ ██████╗██╗ ██╗ ██╔══██╗██║ ██╔═══██╗██╔════╝██║ ██╔╝ @@ -232,18 +253,22 @@ def run(): ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚══════╝ ╚═╝ By Gensyn - """ + """, + style=GENSYN_COLOR, ) if os.environ.get("HF_TOKEN") is None: logging.info("HF_TOKEN not found, prompting") - print( - "Please enter your Hugging Face user access token and press ENTER. If you do not have a token, please refer to" + CONSOLE.print( + "Please enter your Hugging Face user access token and press ENTER. If you do not have a token, please refer to", + style=INFO_COLOR, ) - print() - print("\n https://huggingface.co/docs/hub/en/security-tokens") - print() - print("for instructions on how to obtain one.") + CONSOLE.print() + CONSOLE.print( + "\n https://huggingface.co/docs/hub/en/security-tokens", style=INFO_COLOR + ) + CONSOLE.print() + CONSOLE.print("for instructions on how to obtain one.", style=INFO_COLOR) while True: hf_token = input("Hugging Face token: ").strip() @@ -251,69 +276,82 @@ def run(): break os.environ["HF_TOKEN"] = hf_token - print("✅ HF_TOKEN set successfully") + CONSOLE.print("✅ HF_TOKEN set successfully", style=SUCCESS_COLOR) - print("Setting up virtualenv...") + CONSOLE.print("Setting up virtualenv...", style=LOG_COLOR) setup_venv() - print("Setting up Gradle...") + CONSOLE.print("Setting up Gradle...", style=LOG_COLOR) setup_gradle() - print("Compiling Yarn...") - + CONSOLE.print("Compiling Yarn...", style=LOG_COLOR) setup_yarn() - print("Setting up Minecraft...") + CONSOLE.print("Setting up Minecraft...", style=LOG_COLOR) start_log_watcher() proc_malmo = run_malmo() - print("\nLOGIN") - print("========") + CONSOLE.print(Markdown("# LOGIN"), style=HEADER_COLOR) if sys.platform == "darwin": - print( - "You will likely be asked to approve accessibility permissions. Please do so and, if necessary, restart the program." + CONSOLE.print( + "You will likely be asked to approve accessibility permissions. Please do so and, if necessary, restart the program.", + style=WARNING_COLOR, ) proc_yarn = run_yarn() time.sleep(5) if not os.path.exists("modal-login/temp-data/userData.json"): - print( - "Running Gensyn Testnet login. If browser does not open automatically, please open a browser and go to http://localhost:3000 and click 'login' to continue." + CONSOLE.print( + "Running Gensyn Testnet login. If browser does not open automatically, please open a browser and go to http://localhost:3000 and click 'login' to continue.", + style=INFO_COLOR, + ) + CONSOLE.print( + "Note, if it's your first time playing, also click 'log in')", + style=INFO_COLOR, ) - print("Note, if it's your first time playing, also click 'log in')") run_open() env = wait_for_login() - print("\nSTART MINECRAFT") - print("========") - print( - "Please press ENTER when two Minecraft windows have opened. This may take up to 5 minutes to happen." + CONSOLE.print(Markdown("# START MINECRAFT"), style=HEADER_COLOR) + CONSOLE.print( + "Please press ENTER when two Minecraft windows have opened. This may take up to 5 minutes to happen.", + style=INFO_COLOR, ) - print( - "NOTE: If one or both of the windows closes, please restart the program. You can also `tail -f logs/malmo.log` in another terminal if you suspect an error" + CONSOLE.print( + "NOTE: If one or both of the windows closes, please restart the program. You can also `tail -f logs/malmo.log` in another terminal if you suspect an error", + style=WARNING_COLOR, ) wait_for_enter() - print("Enter received") + CONSOLE.print("Enter received", style=SUCCESS_COLOR) - print("\nINSTRUCTIONS") - print("========") + CONSOLE.print(Markdown("# INSTRUCTIONS"), style=HEADER_COLOR) time.sleep(1) - print("The goal of the game is to build the structure in front of you.") - print("You do this by placing or destroying blocks.") - print("Each building you build is a separate 'episode'") - print("An AI player will assist you.") - print("The more you play, the more the AI player learns.") - print("You should break red blocks and place blocks where indicated") - print("Click on the window and press ENTER to start playing") - print("Left click to break blocks, right click to place blocks") - print( - "Select an axe to break things, or various blocks, by pressing the number keys 1-9" + CONSOLE.print( + "The goal of the game is to build the structure in front of you.", + style=INFO_COLOR, ) - print("Use the WASD keys to move around") - print( - "Once you've finished playing, press ESC, then click back on the terminal window" + CONSOLE.print("You do this by placing or destroying blocks.", style=INFO_COLOR) + CONSOLE.print("Each building you build is a separate 'episode'", style=INFO_COLOR) + CONSOLE.print("An AI player will assist you.", style=INFO_COLOR) + CONSOLE.print("The more you play, the more the AI player learns.", style=INFO_COLOR) + CONSOLE.print( + "You should break red blocks and place blocks where indicated", style=INFO_COLOR + ) + CONSOLE.print( + "Click on the window and press ENTER to start playing", style=INFO_COLOR + ) + CONSOLE.print( + "Left click to break blocks, right click to place blocks", style=INFO_COLOR + ) + CONSOLE.print( + "Select an axe to break things, or various blocks, by pressing the number keys 1-9", + style=INFO_COLOR, + ) + CONSOLE.print("Use the WASD keys to move around", style=INFO_COLOR) + CONSOLE.print( + "Once you've finished playing, press ESC, then click back on the terminal window", + style=INFO_COLOR, ) - print("------\n") proc_blockassist = run_blockassist(env=env) @@ -323,64 +361,66 @@ def run(): for i in range(_MAX_EPISODE_COUNT): # Start timer in a separate thread - print("\nSTARTING EPISODE {}".format(i)) + CONSOLE.print(Markdown(f"\n## STARTING EPISODE {i}"), style=HEADER_COLOR) timer_running = True start_time = time.time() - def timer_display(): - while timer_running: - elapsed = int(time.time() - start_time) - hours = elapsed // 3600 - minutes = (elapsed % 3600) // 60 - seconds = elapsed % 60 - print( - f"\r⏱️ Recording time: {hours:02d}:{minutes:02d}:{seconds:02d}", - end="", - flush=True, - ) - time.sleep(1) + CONSOLE.print( + f"\n[{i}] Please wait for the mission to load up on your Minecraft window. Press ENTER when you have finished recording your episode. **You may have to press it multiple times**", + style=INFO_COLOR, + ) - timer_thread = threading.Thread(target=timer_display, daemon=True) - timer_thread.start() + with Progress( + SpinnerColumn(), + TextColumn("[bold blue]Recording episode"), + BarColumn(), + TaskProgressColumn(), + TimeElapsedColumn(), + console=CONSOLE, + ) as progress: + task = progress.add_task("Recording episode", total=None) - def timer_end(key): - nonlocal timer_running - timer_running = False + wait_for_enter() + progress.stop() - print( - f"\n[{i}] Please wait for the mission to load up on your Minecraft window. Press ENTER when you have finished recording your episode. **You may have to press it multiple times**" - ) - wait_for_enter(timer_end) - print(f"\n[{i}] Enter received") + CONSOLE.print(f"\n[{i}] Enter received", style=SUCCESS_COLOR) - print(f"\n[{i}] Stopping episode recording") + CONSOLE.print(f"\n[{i}] Stopping episode recording", style=INFO_COLOR) send_blockassist_sigint(proc_blockassist.pid) TOTAL_TIME_PLAYED += int(time.time() - start_time) EPISODES_PLAYED += 1 - print("Stopping Malmo") + CONSOLE.print("Stopping Malmo", style=INFO_COLOR) proc_malmo.kill() proc_malmo.wait() - print("Waiting for BlockAssist to stop - this might take a few minutes") + CONSOLE.print( + "Waiting for BlockAssist to stop - this might take a few minutes", + style=INFO_COLOR, + ) proc_blockassist.wait() - print("\nMODEL TRAINING") - print("========") - print("Your assistant is now training on the gameplay you recorded.") - print( - "This may take a while, depending on your hardware. Please keep this window open until you see 'Training complete'." + CONSOLE.print(Markdown("# MODEL TRAINING"), style=HEADER_COLOR) + CONSOLE.print( + "Your assistant is now training on the gameplay you recorded.", style=INFO_COLOR + ) + CONSOLE.print( + "This may take a while, depending on your hardware. Please keep this window open until you see 'Training complete'.", + style=INFO_COLOR, ) - print("Running training") + CONSOLE.print("Running training", style=INFO_COLOR) proc_train = train_blockassist(env=env) proc_train.wait() - print("Training complete") + CONSOLE.print("Training complete", style=SUCCESS_COLOR) - print("\nUPLOAD TO HUGGINGFACE AND SMART CONTRACT") - print("========") + CONSOLE.print( + Markdown("# UPLOAD TO HUGGINGFACE AND SMART CONTRACT"), style=HEADER_COLOR + ) # Monitor blockassist-train.log for HuggingFace upload confirmation and transaction hash - print("Checking for upload confirmation and transaction hash...") + CONSOLE.print( + "Checking for upload confirmation and transaction hash...", style=LOG_COLOR + ) train_log_path = "logs/blockassist-train.log" upload_confirmed = False transaction_hash = None @@ -409,66 +449,85 @@ def timer_end(key): )[1].split(" ") hf_path = line_elems[0].strip() hf_size = line_elems[3].strip() + " " + line_elems[4].strip() - print("✅ " + line) + CONSOLE.print("✅ " + line, style=SUCCESS_COLOR) upload_confirmed = True elif "HF Upload API response:" in line and not transaction_hash: - print("🔗 " + line) + CONSOLE.print("🔗 " + line, style=WARNING_COLOR) transaction_hash = line # If we found both, we can stop monitoring if upload_confirmed and transaction_hash: - print( - "Copy your HuggingFace model path (e.g. 'block-fielding/bellowing_pouncing_horse_1753796381') and check for it here:\nhttps://gensyn-testnet.explorer.alchemy.com/address/0xE2070109A0C1e8561274E59F024301a19581d45c?tab=logs" + CONSOLE.print( + "Copy your HuggingFace model path (e.g. 'block-fielding/bellowing_pouncing_horse_1753796381') and check for it here:\nhttps://gensyn-testnet.explorer.alchemy.com/address/0xE2070109A0C1e8561274E59F024301a19581d45c?tab=logs", + style=INFO_COLOR, ) break except Exception as e: - print(f"Error reading log file: {e}") + CONSOLE.print(f"Error reading log file: {e}", style=ERROR_COLOR) break # If we didn't find the logs after 30 seconds if not upload_confirmed and not transaction_hash: - print( - "⚠️ No upload confirmation or transaction hash found in blockassist-train.log" + CONSOLE.print( + "⚠️ No upload confirmation or transaction hash found in blockassist-train.log", + style=ERROR_COLOR, ) elif not upload_confirmed: - print("⚠️ No HuggingFace upload confirmation found in blockassist-train.log") + CONSOLE.print( + "⚠️ No HuggingFace upload confirmation found in blockassist-train.log", + style=ERROR_COLOR, + ) elif not transaction_hash: - print("⚠️ No transaction hash found in blockassist-train.log") + CONSOLE.print( + "⚠️ No transaction hash found in blockassist-train.log", style=ERROR_COLOR + ) - print("\nSHUTTING DOWN") - print("========") - print("Stopping Yarn") + CONSOLE.print(Markdown("# SHUTTING DOWN"), style=HEADER_COLOR) + CONSOLE.print("Stopping Yarn", style=LOG_COLOR) proc_yarn.kill() - print(f"🎉 SUCCESS! Your BlockAssist session has completed successfully!") - print(f"") - print(f"- Your gameplay was recorded and analyzed") - print(f"- An AI model was trained on your building patterns") - print(f"- The model was successfully uploaded to Hugging Face") - print(f"- Your work helps train better AI assistants ") - print(f"") - print(f"Stats:") - print(f"") - print(f"- Episodes recorded: {EPISODES_PLAYED}") - print( - f"- Total gameplay time: {TOTAL_TIME_PLAYED // 60}m {TOTAL_TIME_PLAYED % 60}s" + CONSOLE.print( + Markdown(f"# 🎉 SUCCESS! Your BlockAssist session has completed successfully!"), + style=SUCCESS_COLOR, + ) + CONSOLE.print(f"") + CONSOLE.print(f"- Your gameplay was recorded and analyzed", style=INFO_COLOR) + CONSOLE.print( + f"- An AI model was trained on your building patterns", style=INFO_COLOR + ) + CONSOLE.print( + f"- The model was successfully uploaded to Hugging Face", style=INFO_COLOR + ) + CONSOLE.print(f"- Your work helps train better AI assistants ", style=INFO_COLOR) + CONSOLE.print(f"") + CONSOLE.print(f"Stats:", style=INFO_COLOR) + CONSOLE.print(f"") + CONSOLE.print(f"- Episodes recorded: {EPISODES_PLAYED}", style=INFO_COLOR) + CONSOLE.print( + f"- Total gameplay time: {TOTAL_TIME_PLAYED // 60}m {TOTAL_TIME_PLAYED % 60}s", + style=INFO_COLOR, + ) + CONSOLE.print(f"- Model trained and uploaded: {hf_path}", style=INFO_COLOR) + CONSOLE.print(f"- Model size: {hf_size}", style=INFO_COLOR) + CONSOLE.print(f"") + CONSOLE.print(f"🚀What to do next:", style=INFO_COLOR) + CONSOLE.print(f"") + CONSOLE.print(f"") + CONSOLE.print( + f"- Run BlockAssist again to improve your performance (higher completion %, faster time).", + style=INFO_COLOR, ) - print(f"- Model trained and uploaded: {hf_path}") - print(f"- Model size: {hf_size}") - print(f"") - print(f"🚀What to do next:") - print(f"") - print(f"") - print( - f"- Run BlockAssist again to improve your performance (higher completion %, faster time)." + CONSOLE.print( + f"- Check your model on Hugging Face: https://huggingface.co/{hf_path}", + style=INFO_COLOR, ) - print(f"- Check your model on Hugging Face: https://huggingface.co/{hf_path}") - print( - f"- Screenshot your stats, record your gameplay, and share with the community on X (https://x.com/gensynai) or Discord (https://discord.gg/gensyn)" + CONSOLE.print( + f"- Screenshot your stats, record your gameplay, and share with the community on X (https://x.com/gensynai) or Discord (https://discord.gg/gensyn)", + style=INFO_COLOR, ) - print(f"") - print(f"Thank you for contributing to BlockAssist!") + CONSOLE.print(f"") + CONSOLE.print(f"Thank you for contributing to BlockAssist!", style=INFO_COLOR) if __name__ == "__main__": diff --git a/scripts/gradle_setup.sh b/scripts/gradle_setup.sh index ab6cc3a..2825761 100755 --- a/scripts/gradle_setup.sh +++ b/scripts/gradle_setup.sh @@ -6,13 +6,13 @@ set -o pipefail mkdir -p ~/.gradle if [ ! -f "~/.gradle/gradle.properties" ]; then - echo "Creating ~/.gradle/gradle.properties" + echo "Creating ~/.gradle/gradle.properties" >> logs/gradle_setup.log touch ~/.gradle/gradle.properties fi -if grep -Fxq "org.gradle.daemon=true" "$HOME/.gradle/gradle.properties"; then - echo "Gradle daemon setting already exists" +if grep -Fxq "org.gradle.daemon=true" "$HOME/.gradle/gradle.properties"; then + echo "Gradle daemon setting already exists" >> logs/gradle_setup.log else - echo "Adding gradle daemon setting" + echo "Adding gradle daemon setting" >> logs/gradle_setup.log echo "org.gradle.daemon=true" >> ~/.gradle/gradle.properties; fi diff --git a/scripts/node_env.sh b/scripts/node_env.sh index 7033429..4e50357 100755 --- a/scripts/node_env.sh +++ b/scripts/node_env.sh @@ -11,10 +11,10 @@ fi # Function to setup Node.js and NVM setup_node_nvm() { - echo "Setting up Node.js and NVM..." + echo "Setting up Node.js and NVM..." >> logs/node_env.log if ! command -v node > /dev/null 2>&1; then - echo "Node.js not found. Installing NVM and latest Node.js..." + echo "Node.js not found. Installing NVM and latest Node.js..." >> logs/node_env.log export NVM_DIR="$HOME/.nvm" if [ ! -d "$NVM_DIR" ]; then curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash @@ -23,18 +23,19 @@ setup_node_nvm() { [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" nvm install node else - echo "Node.js is already installed: $(node -v)" + echo "Node.js is already installed: $(node -v)" >> logs/node_env.log fi if ! command -v yarn > /dev/null 2>&1; then # Detect Ubuntu (including WSL Ubuntu) and install Yarn accordingly if grep -qi "ubuntu" /etc/os-release 2> /dev/null || uname -r | grep -qi "microsoft"; then - echo "Detected Ubuntu or WSL Ubuntu. Installing Yarn via apt..." + echo "Detected Ubuntu or WSL Ubuntu. Installing Yarn via apt..." >> logs/node_env.log + curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list sudo apt update && sudo apt install -y yarn else - echo "Yarn not found. Installing Yarn globally with npm (no profile edits)…" + echo "Yarn not found. Installing Yarn globally with npm (no profile edits)…" >> logs/node_env.log # This lands in $NVM_DIR/versions/node//bin which is already on PATH npm install -g --silent yarn fi @@ -42,7 +43,7 @@ setup_node_nvm() { } # Function to setup environment file setup_environment() { - echo "Setting up environment configuration..." + echo "Setting up environment configuration..." >> logs/node_env.log ENV_FILE="$PWD"/.env if [[ "$OSTYPE" == "darwin"* ]]; then @@ -53,5 +54,5 @@ setup_environment() { sed -i "3s/.*/SMART_CONTRACT_ADDRESS=$SMART_CONTRACT_ADDRESS/" "$ENV_FILE" fi - echo "Environment file updated with SMART_CONTRACT_ADDRESS: $SMART_CONTRACT_ADDRESS" + echo "Environment file updated with SMART_CONTRACT_ADDRESS: $SMART_CONTRACT_ADDRESS" >> logs/node_env.log } \ No newline at end of file diff --git a/scripts/yarn_run.sh b/scripts/yarn_run.sh index 8aeae88..9688080 100755 --- a/scripts/yarn_run.sh +++ b/scripts/yarn_run.sh @@ -11,9 +11,9 @@ fi ROOT="$PWD" # Kill any existing processes on port 3000 -echo "Checking for existing processes on port 3000..." +echo "Checking for existing processes on port 3000..." >> logs/yarn_run.log if lsof -ti :3000 > /dev/null 2>&1; then - echo "Killing existing processes on port 3000..." + echo "Killing existing processes on port 3000..." >> logs/yarn_run.log kill $(lsof -ti :3000) 2>/dev/null || true sleep 2 fi diff --git a/scripts/yarn_setup.sh b/scripts/yarn_setup.sh index 326d852..38a9419 100755 --- a/scripts/yarn_setup.sh +++ b/scripts/yarn_setup.sh @@ -16,10 +16,10 @@ if [ ! -d .next ]; then ### Setup environment configuration setup_environment - echo "Installing dependencies..." + echo "Installing dependencies..." >> logs/yarn_setup.log yarn install --immutable - echo "Building server..." + echo "Building server..." >> logs/yarn_setup.log yarn build fi \ No newline at end of file From 341faf26f45f440f2d2737fb33445ce6ace1f778 Mon Sep 17 00:00:00 2001 From: Eamonn Nugent <4699217+space55@users.noreply.github.com> Date: Mon, 8 Sep 2025 16:41:41 -0400 Subject: [PATCH 04/11] Some fixes --- modal-login/app/api/submit-hf-upload/route.ts | 22 ++--- modal-login/app/lib/contract.json | 83 ++++++++++++++++++- src/blockassist/blockchain/coordinator.py | 19 +++-- src/blockassist/distributed/hf.py | 51 ++++++++---- src/blockassist/launch.py | 3 +- 5 files changed, 144 insertions(+), 34 deletions(-) diff --git a/modal-login/app/api/submit-hf-upload/route.ts b/modal-login/app/api/submit-hf-upload/route.ts index 0461c98..3a18bd2 100644 --- a/modal-login/app/api/submit-hf-upload/route.ts +++ b/modal-login/app/api/submit-hf-upload/route.ts @@ -9,12 +9,13 @@ export async function POST(request: Request) { huggingFaceId: string; numSessions: bigint; telemetryEnabled: boolean; + gitRef: string; } = await request.json().catch((err) => { console.error(err); console.log(body); return NextResponse.json( - { error: " bad request generic "}, + { error: " bad request generic " }, { status: 400 }, ); }) @@ -23,7 +24,7 @@ export async function POST(request: Request) { if (!body.orgId) { return NextResponse.json( - { error: " bad request orgId "}, + { error: " bad request orgId " }, { status: 400 }, ); } @@ -32,28 +33,28 @@ export async function POST(request: Request) { const user = getUser(body.orgId); if (!user) { return NextResponse.json( - { error: " user not found "}, + { error: " user not found " }, { status: 404 }, ); } const apiKey = getLatestApiKey(body.orgId); console.log("API Key retrieved:", apiKey ? { activated: apiKey.activated, hasFields: apiKey.activated ? !!apiKey.accountAddress : false } : "null"); - + if (!apiKey?.activated) { return NextResponse.json( - { error: " api key not found or not activated "}, + { error: " api key not found or not activated " }, { status: 500 }, ); } const { accountAddress, privateKey, initCode, deferredActionDigest } = apiKey; - + // Validate that all required fields are present if (!accountAddress || !privateKey || !initCode || !deferredActionDigest) { console.error("Missing required API key fields:", { accountAddress, privateKey, initCode, deferredActionDigest }); return NextResponse.json( - { error: " api key missing required fields "}, + { error: " api key missing required fields " }, { status: 500 }, ); } @@ -70,7 +71,8 @@ export async function POST(request: Request) { trainingId: body.trainingId, huggingFaceId: body.huggingFaceId, numSessions: body.numSessions, - telemetryEnabled: body.telemetryEnabled + telemetryEnabled: body.telemetryEnabled, + gitRef: body.gitRef }); const userOperationResponse = await userOperationHandler({ @@ -78,8 +80,8 @@ export async function POST(request: Request) { privateKey, deferredActionDigest, initCode, - functionName: "submitHFUpload", - args: [accountAddress, body.trainingId, body.huggingFaceId, body.numSessions, body.telemetryEnabled], + functionName: "submitHFUploadWithRef", + args: [accountAddress, body.trainingId, body.huggingFaceId, body.gitRef, body.numSessions, body.telemetryEnabled], }); return userOperationResponse; diff --git a/modal-login/app/lib/contract.json b/modal-login/app/lib/contract.json index 5b2a5cd..3c13659 100644 --- a/modal-login/app/lib/contract.json +++ b/modal-login/app/lib/contract.json @@ -1,5 +1,5 @@ { - "abi": [ + "abi": [ { "inputs": [ { @@ -579,6 +579,87 @@ "outputs": [], "stateMutability": "nonpayable", "type": "function" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "user", + "type": "address" + }, + { + "indexed": false, + "internalType": "string", + "name": "trainingID", + "type": "string" + }, + { + "indexed": false, + "internalType": "string", + "name": "huggingFaceID", + "type": "string" + }, + { + "indexed": false, + "internalType": "string", + "name": "gitRef", + "type": "string" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "numSessions", + "type": "uint256" + }, + { + "indexed": false, + "internalType": "bool", + "name": "telemetryEnabled", + "type": "bool" + } + ], + "name": "HFUploadSubmittedWithRef", + "type": "event" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "user", + "type": "address" + }, + { + "internalType": "string", + "name": "trainingID", + "type": "string" + }, + { + "internalType": "string", + "name": "huggingFaceID", + "type": "string" + }, + { + "internalType": "string", + "name": "gitRef", + "type": "string" + }, + { + "internalType": "uint256", + "name": "numSessions", + "type": "uint256" + }, + { + "internalType": "bool", + "name": "telemetryEnabled", + "type": "bool" + } + ], + "name": "submitHFUploadWithRef", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" } ] } \ No newline at end of file diff --git a/src/blockassist/blockchain/coordinator.py b/src/blockassist/blockchain/coordinator.py index 4a89703..4cbb9ee 100644 --- a/src/blockassist/blockchain/coordinator.py +++ b/src/blockassist/blockchain/coordinator.py @@ -6,16 +6,25 @@ logger = logging.getLogger(__name__) -class ModalSwarmCoordinator(): + +class ModalSwarmCoordinator: def __init__(self, org_id: str) -> None: self.org_id = org_id - def submit_hf_upload(self, training_id, hf_id, num_sessions, telemetry_enabled): + def submit_hf_upload( + self, training_id, hf_id, num_sessions, telemetry_enabled, git_ref + ): try: send_via_api( self.org_id, "submit-hf-upload", - {"trainingId": training_id, "huggingFaceId": hf_id, "numSessions": num_sessions, "telemetryEnabled": telemetry_enabled} + { + "trainingId": training_id, + "huggingFaceId": hf_id, + "numSessions": num_sessions, + "telemetryEnabled": telemetry_enabled, + "gitRef": git_ref, + }, ) except requests.exceptions.HTTPError as e: if e.response is None or e.response.status_code != 500: @@ -32,5 +41,5 @@ def send_via_api(org_id, method, args): # Send the POST request. response = requests.post(url, json=payload) response.raise_for_status() # Raise an exception for HTTP errors - logger.info('HF Upload API response: %s', response.json()) - return response.json() \ No newline at end of file + logger.info("HF Upload API response: %s", response.json()) + return response.json() diff --git a/src/blockassist/distributed/hf.py b/src/blockassist/distributed/hf.py index 09849e4..28cc1f7 100644 --- a/src/blockassist/distributed/hf.py +++ b/src/blockassist/distributed/hf.py @@ -9,20 +9,16 @@ _LOG = get_logger() + def _create_readme(model_path: Path, user_id: str | None = None) -> None: readme_path = model_path / "README.md" - + # Generate identifier tag from address if provided - tags = [ - "gensyn", - "blockassist", - "gensyn-blockassist", - "minecraft" - ] + tags = ["gensyn", "blockassist", "gensyn-blockassist", "minecraft"] if user_id: tags.append(user_id.replace("_", " ")) - + # Create YAML front matter with tags tags_yaml = "\n".join(f" - {tag}" for tag in tags) front_matter = f"""\ @@ -38,16 +34,33 @@ def _create_readme(model_path: Path, user_id: str | None = None) -> None: Gensyn's BlockAssist is a distributed extension of the paper [AssistanceZero: Scalably Solving Assistance Games](https://arxiv.org/abs/2504.07091). """ readme_path.write_text(front_matter + body, encoding="utf-8") - _LOG.info(f"Created README.md with YAML metadata and tags {tags} at {readme_path!r}") + _LOG.info( + f"Created README.md with YAML metadata and tags {tags} at {readme_path!r}" + ) + def upload_to_huggingface( - model_path: Path, - user_id: str, - repo_id: str, - hf_token: str | None = None, + model_path: Path, + user_id: str, + repo_id: str, + hf_token: str | None = None, chain_metadata_dict: dict | None = None, address_eoa: str | None = None, -) -> None: +) -> str: + """Uploads the model directory to HuggingFace. + Args: + model_path (Path): Path to the model directory. + user_id (str): Identifier for the user (e.g., address). + repo_id (str): HuggingFace repository ID. + hf_token (str | None): HuggingFace authentication token. + chain_metadata_dict (dict | None): Optional metadata dictionary to upload. + address_eoa (str | None): Optional EOA address for additional context. + Returns: + str: The git reference of the uploaded model. + Raises: + FileNotFoundError: If the model directory does not exist. + Exception: If the upload fails. + """ if not model_path.exists(): raise FileNotFoundError(f"Model directory does not exist: {model_path}") @@ -55,7 +68,9 @@ def upload_to_huggingface( _create_readme(model_path, user_id=user_id) api = HfApi(token=hf_token) api.create_repo(repo_id=repo_id, repo_type="model", exist_ok=True) - api.upload_folder(repo_id=repo_id, repo_type="model", folder_path=model_path) + ret = api.upload_folder( + repo_id=repo_id, repo_type="model", folder_path=model_path + ) if chain_metadata_dict: metadata_json = json.dumps(chain_metadata_dict, indent=2) @@ -64,17 +79,19 @@ def upload_to_huggingface( path_or_fileobj=metadata_bytes, path_in_repo="gensyn.json", repo_id=repo_id, - repo_type="model" + repo_type="model", ) _LOG.info("Uploaded metadata dictionary") # Calculate total size total_size = sum(f.stat().st_size for f in model_path.rglob("*") if f.is_file()) _LOG.info( - f"Successfully uploaded model to HuggingFace: {repo_id} with size {total_size / 1024 / 1024:.2f} MB" + f"Successfully uploaded model to HuggingFace: {repo_id} with size {total_size / 1024 / 1024:.2f} MB ({ret.oid})" ) telemetry.push_telemetry_event_uploaded(total_size, user_id, repo_id) + return ret.oid + except Exception as e: _LOG.error(f"Failed to upload model to HuggingFace: {e}", exc_info=True) raise diff --git a/src/blockassist/launch.py b/src/blockassist/launch.py index 3ebbdaa..36d407e 100644 --- a/src/blockassist/launch.py +++ b/src/blockassist/launch.py @@ -169,7 +169,7 @@ async def _main(cfg: DictConfig): hf_repo_id = get_hf_repo_id(hf_token, training_id) num_sessions = get_total_episodes(checkpoint_dir) is_telemetry_enabled = not telemetry.is_telemetry_disabled() - upload_to_huggingface( + git_ref = upload_to_huggingface( model_path=Path(model_dir), user_id=get_identifier(address_eoa), repo_id=hf_repo_id, @@ -186,6 +186,7 @@ async def _main(cfg: DictConfig): hf_id=hf_repo_id, num_sessions=num_sessions, telemetry_enabled=is_telemetry_enabled, + git_ref=git_ref, ) else: _LOG.warning("No model directory specified, skipping upload.") From 166f028ed45291cff057cb29bb88ed52e0d9c132 Mon Sep 17 00:00:00 2001 From: Eamonn Nugent <4699217+space55@users.noreply.github.com> Date: Tue, 9 Sep 2025 12:12:26 -0400 Subject: [PATCH 05/11] Update version to 0.0.2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 86673a3..8348089 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ name = "blockassist" description = "Distributed experiments built using the MBAG multiagent environment (https://github.com/cassidylaidlaw/minecraft-building-assistance-game)." readme = "README.md" requires-python = ">=3.10,<4.0" -version = "0.0.1" +version = "0.0.2" dependencies = [ "aiofiles", "boto3", From 9b415d97391bbd58f6c1383e3d98d783fec5c087 Mon Sep 17 00:00:00 2001 From: Eamonn Nugent <4699217+space55@users.noreply.github.com> Date: Tue, 9 Sep 2025 12:28:05 -0400 Subject: [PATCH 06/11] Bump to 0.0.3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8348089..7ae94b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ name = "blockassist" description = "Distributed experiments built using the MBAG multiagent environment (https://github.com/cassidylaidlaw/minecraft-building-assistance-game)." readme = "README.md" requires-python = ">=3.10,<4.0" -version = "0.0.2" +version = "0.0.3" dependencies = [ "aiofiles", "boto3", From 7b25b479e3597f572f448b2f441246d9ad6ef17d Mon Sep 17 00:00:00 2001 From: Eamonn Nugent <4699217+space55@users.noreply.github.com> Date: Tue, 9 Sep 2025 12:29:17 -0400 Subject: [PATCH 07/11] Bump to 0.1.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7ae94b2..46d946a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ name = "blockassist" description = "Distributed experiments built using the MBAG multiagent environment (https://github.com/cassidylaidlaw/minecraft-building-assistance-game)." readme = "README.md" requires-python = ">=3.10,<4.0" -version = "0.0.3" +version = "0.1.0" dependencies = [ "aiofiles", "boto3", From 606b3e9665625c87faf3f8798659d1f1679b18f7 Mon Sep 17 00:00:00 2001 From: Eamonn Nugent <4699217+space55@users.noreply.github.com> Date: Wed, 10 Sep 2025 13:53:01 -0400 Subject: [PATCH 08/11] Fix for participation --- pyproject.toml | 2 +- src/blockassist/distributed/hf.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 46d946a..b2ae656 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ name = "blockassist" description = "Distributed experiments built using the MBAG multiagent environment (https://github.com/cassidylaidlaw/minecraft-building-assistance-game)." readme = "README.md" requires-python = ">=3.10,<4.0" -version = "0.1.0" +version = "0.1.1" dependencies = [ "aiofiles", "boto3", diff --git a/src/blockassist/distributed/hf.py b/src/blockassist/distributed/hf.py index 28cc1f7..4b866dd 100644 --- a/src/blockassist/distributed/hf.py +++ b/src/blockassist/distributed/hf.py @@ -68,14 +68,13 @@ def upload_to_huggingface( _create_readme(model_path, user_id=user_id) api = HfApi(token=hf_token) api.create_repo(repo_id=repo_id, repo_type="model", exist_ok=True) - ret = api.upload_folder( - repo_id=repo_id, repo_type="model", folder_path=model_path - ) + api.upload_folder(repo_id=repo_id, repo_type="model", folder_path=model_path) + ret = None if chain_metadata_dict: metadata_json = json.dumps(chain_metadata_dict, indent=2) metadata_bytes = io.BytesIO(metadata_json.encode("utf-8")) - api.upload_file( + ret = api.upload_file( path_or_fileobj=metadata_bytes, path_in_repo="gensyn.json", repo_id=repo_id, @@ -90,7 +89,9 @@ def upload_to_huggingface( ) telemetry.push_telemetry_event_uploaded(total_size, user_id, repo_id) - return ret.oid + if ret: + return ret.oid + return None except Exception as e: _LOG.error(f"Failed to upload model to HuggingFace: {e}", exc_info=True) From 3ad771226b038c0b4719b95919a21e79c8026103 Mon Sep 17 00:00:00 2001 From: Eamonn Nugent <4699217+space55@users.noreply.github.com> Date: Thu, 2 Oct 2025 10:58:53 -0400 Subject: [PATCH 09/11] Change the animal name to EOA in internal values --- pyproject.toml | 2 +- src/blockassist/blockchain/names.py | 95 ----------------------------- src/blockassist/distributed/hf.py | 8 +-- src/blockassist/episode.py | 3 +- src/blockassist/globals.py | 10 ++- src/blockassist/launch.py | 8 +-- src/blockassist/train.py | 3 +- 7 files changed, 15 insertions(+), 114 deletions(-) delete mode 100644 src/blockassist/blockchain/names.py diff --git a/pyproject.toml b/pyproject.toml index b2ae656..0b3cd55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ name = "blockassist" description = "Distributed experiments built using the MBAG multiagent environment (https://github.com/cassidylaidlaw/minecraft-building-assistance-game)." readme = "README.md" requires-python = ">=3.10,<4.0" -version = "0.1.1" +version = "0.1.2" dependencies = [ "aiofiles", "boto3", diff --git a/src/blockassist/blockchain/names.py b/src/blockassist/blockchain/names.py deleted file mode 100644 index ae5423c..0000000 --- a/src/blockassist/blockchain/names.py +++ /dev/null @@ -1,95 +0,0 @@ -import hashlib -from functools import lru_cache -from typing import Sequence - -# fmt: off -ADJECTIVES = [ - "agile", "alert", "amphibious", "aquatic", "arctic", "armored", "barky", "beaked", - "bellowing", "bipedal", "bold", "bristly", "burrowing", "camouflaged", "carnivorous", "chattering", - "clawed", "climbing", "coiled", "colorful", "crested", "cunning", "curious", "dappled", - "darting", "deadly", "deft", "dense", "dextrous", "diving", "docile", "domestic", - "dormant", "downy", "durable", "eager", "elusive", "endangered", "energetic", "enormous", - "exotic", "extinct", "fanged", "fast", "feathered", "feline", "ferocious", "fierce", - "finicky", "fishy", "flapping", "fleecy", "flexible", "flightless", "fluffy", "foraging", - "foxy", "freckled", "frisky", "furry", "galloping", "gentle", "giant", "gilded", - "gliding", "graceful", "grassy", "grazing", "gregarious", "grunting", "hairy", "hardy", - "hibernating", "hoarse", "horned", "howling", "huge", "hulking", "humming", "hunting", - "insectivorous", "invisible", "iridescent", "jagged", "jumping", "keen", "knobby", "lanky", - "large", "lazy", "leaping", "leggy", "lethal", "lightfooted", "lithe", "lively", - "long", "loud", "lumbering", "majestic", "mammalian", "mangy", "marine", "masked", - "meek", "melodic", "mighty", "mimic", "miniature", "moist", "monstrous", "mottled", - "muscular", "mute", "nasty", "nimble", "nocturnal", "noisy", "omnivorous", "opaque", - "padded", "pale", "patterned", "pawing", "peaceful", "peckish", "pensive", "pesty", - "placid", "playful", "plump", "poisonous", "polished", "pouncing", "powerful", "prehistoric", - "prickly", "prowling", "pudgy", "purring", "quick", "quiet", "rabid", "raging", - "rangy", "rapid", "ravenous", "reclusive", "regal", "reptilian", "restless", "roaring", - "robust", "rough", "rugged", "running", "savage", "scaly", "scampering", "scavenging", - "scented", "screeching", "scruffy", "scurrying", "secretive", "sedate", "shaggy", "sharp", - "shiny", "short", "shrewd", "shy", "silent", "silky", "singing", "sizable", - "skilled", "skittish", "sleek", "slender", "slimy", "slithering", "slow", "sly", - "small", "smooth", "snappy", "sneaky", "sniffing", "snorting", "soaring", "soft", - "solitary", "spotted", "sprightly", "squeaky", "squinting", "stalking", "stealthy", "stinging", - "stinky", "stocky", "striped", "strong", "stubby", "sturdy", "subtle", "swift", - "tall", "tame", "tangled", "tawny", "tenacious", "territorial", "thick", "thorny", - "thriving", "timid", "tiny", "toothy", "tough", "tricky", "tropical", "trotting", - "twitchy", "unseen", "untamed", "vicious", "vigilant", "vocal", "voracious", "waddling", - "wary", "webbed", "whiskered", "whistling", "wild", "wily", "winged", "wiry", - "wise", "woolly", "yapping", "yawning", "zealous" -] - -ANIMALS = [ - "aardvark", "albatross", "alligator", "alpaca", "anaconda", "ant", "anteater", "antelope", - "ape", "armadillo", "baboon", "badger", "barracuda", "bat", "bear", "beaver", - "bee", "bison", "boar", "bobcat", "buffalo", "butterfly", "camel", "capybara", - "caribou", "cassowary", "cat", "caterpillar", "cheetah", "chicken", "chameleon", "chimpanzee", - "chinchilla", "clam", "cobra", "cockroach", "cod", "condor", "coral", "cougar", "cow", - "coyote", "crab", "crane", "crocodile", "crow", "deer", "dingo", "dinosaur", - "dog", "dolphin", "donkey", "dove", "dragonfly", "duck", "eagle", "eel", - "elephant", "elk", "emu", "falcon", "ferret", "finch", "fish", "flamingo", - "flea", "fly", "fox", "frog", "gazelle", "gecko", "gerbil", "gibbon", - "giraffe", "goat", "goose", "gorilla", "grasshopper", "grouse", "gull", "hamster", - "hare", "hawk", "hedgehog", "heron", "hippo", "hornet", "horse", "hummingbird", - "hyena", "ibis", "iguana", "impala", "jackal", "jaguar", "jay", "jellyfish", - "kangaroo", "kingfisher", "kiwi", "koala", "komodo", "ladybug", "lemur", "leopard", - "lion", "lizard", "llama", "lobster", "locust", "lynx", "macaque", "macaw", - "magpie", "mallard", "mammoth", "manatee", "mandrill", "mantis", "marmot", "meerkat", - "mink", "mole", "mongoose", "monkey", "moose", "mosquito", "mouse", "mule", - "narwhal", "newt", "nightingale", "ocelot", "octopus", "okapi", "opossum", "orangutan", "ostrich", - "otter", "owl", "ox", "panda", "panther", "parrot", "peacock", "pelican", - "penguin", "pheasant", "pig", "pigeon", "piranha", "platypus", "porcupine", "porpoise", - "prawn", "puffin", "puma", "python", "quail", "rabbit", "raccoon", - "ram", "rat", "raven", "reindeer", "rhino", "robin", "rooster", "salamander", - "salmon", "sandpiper", "sardine", "scorpion", "seahorse", "seal", "sealion", - "shark", "sheep", "shrew", "shrimp", "skunk", "sloth", "slug", "snail", - "snake", "sparrow", "spider", "squid", "squirrel", "starfish", "stingray", "stork", - "swan", "tamarin", "tapir", "tarantula", "termite", "tiger", "toad", "tortoise", - "toucan", "trout", "tuna", "turkey", "turtle", "viper", "vulture", "wallaby", - "walrus", "warthog", "wasp", "weasel", "whale", "wildebeest", "wolf", "wombat", - "woodpecker", "worm", "yak", "zebra" -] -# fmt: on - - -def hex_to_ints(s, k): - """Converts hex-encoded strings to int lists. Specify chunk size with k.""" - return tuple(int(s[i : i + k], 16) for i in range(0, len(s), k)) - -@lru_cache -def get_name_from_str(s: str, no_spaces=False): - # ~200 entries for both lists; so 2 hex digits. - ints = hex_to_ints(hashlib.md5(s.encode()).hexdigest(), 2) - adj1 = ADJECTIVES[ints[2] % len(ADJECTIVES)] - adj2 = ADJECTIVES[ints[1] % len(ADJECTIVES)] - animal = ANIMALS[ints[0] % len(ANIMALS)] - - name = f"{adj1} {adj2} {animal}" - if no_spaces: - name = "_".join(name.split(" ")) - return name - - -def search_for_name(strs: Sequence[str], name): - for s in strs: - if name == get_name_from_str(s): - return s - return None \ No newline at end of file diff --git a/src/blockassist/distributed/hf.py b/src/blockassist/distributed/hf.py index 4b866dd..35160c7 100644 --- a/src/blockassist/distributed/hf.py +++ b/src/blockassist/distributed/hf.py @@ -5,7 +5,7 @@ from huggingface_hub import HfApi from blockassist import telemetry -from blockassist.globals import get_identifier, get_logger +from blockassist.globals import get_logger _LOG = get_logger() @@ -20,7 +20,7 @@ def _create_readme(model_path: Path, user_id: str | None = None) -> None: tags.append(user_id.replace("_", " ")) # Create YAML front matter with tags - tags_yaml = "\n".join(f" - {tag}" for tag in tags) + tags_yaml = "\n".join(f' - "{tag}"' for tag in tags) front_matter = f"""\ --- tags: @@ -43,9 +43,9 @@ def upload_to_huggingface( model_path: Path, user_id: str, repo_id: str, + address_eoa: str, hf_token: str | None = None, chain_metadata_dict: dict | None = None, - address_eoa: str | None = None, ) -> str: """Uploads the model directory to HuggingFace. Args: @@ -87,7 +87,7 @@ def upload_to_huggingface( _LOG.info( f"Successfully uploaded model to HuggingFace: {repo_id} with size {total_size / 1024 / 1024:.2f} MB ({ret.oid})" ) - telemetry.push_telemetry_event_uploaded(total_size, user_id, repo_id) + telemetry.push_telemetry_event_uploaded(total_size, address_eoa, repo_id) if ret: return ret.oid diff --git a/src/blockassist/episode.py b/src/blockassist/episode.py index 966d5ec..ff99837 100644 --- a/src/blockassist/episode.py +++ b/src/blockassist/episode.py @@ -10,7 +10,6 @@ from blockassist.globals import ( _DEFAULT_CHECKPOINT, _MAX_EPISODE_COUNT, - get_identifier, get_logger, ) from blockassist.goals.generator import BlockAssistGoalGenerator @@ -104,7 +103,7 @@ def after_episode(self, result): duration_ms = int((time.time() - self.start_time) * 1000) telemetry.push_telemetry_event_session( - duration_ms, get_identifier(self.address_eoa), self.get_last_goal_percentage_min(result) + duration_ms, self.address_eoa, self.get_last_goal_percentage_min(result) ) def before_session(self): diff --git a/src/blockassist/globals.py b/src/blockassist/globals.py index 5364521..d1b301d 100644 --- a/src/blockassist/globals.py +++ b/src/blockassist/globals.py @@ -2,8 +2,6 @@ import socket import time -from blockassist.blockchain.names import get_name_from_str - _DATA_DIR = "data" _DEFAULT_CHECKPOINT = f"{_DATA_DIR}/base_checkpoint" _DEFAULT_EPISODES_S3_BUCKET = "blockassist-episode" @@ -23,14 +21,14 @@ def get_logger() -> logging.Logger: lib_logger.addHandler(logging.NullHandler()) return _LOG + def get_hostname() -> str: return socket.gethostname() -def get_ip(hostname = get_hostname()) -> str: + +def get_ip(hostname=get_hostname()) -> str: return socket.gethostbyname(hostname) -def get_identifier(address_eoa: str) -> str: - return get_name_from_str(address_eoa).replace(" ", "_") def get_training_id(address_eoa: str) -> str: - return f"{get_identifier(address_eoa)}_{int(time.time())}" \ No newline at end of file + return f"{address_eoa}_{int(time.time())}" diff --git a/src/blockassist/launch.py b/src/blockassist/launch.py index 36d407e..1ab63b7 100644 --- a/src/blockassist/launch.py +++ b/src/blockassist/launch.py @@ -30,7 +30,6 @@ from blockassist.globals import ( _DEFAULT_CHECKPOINT, _DEFAULT_EPISODES_S3_BUCKET, - get_identifier, get_logger, get_training_id, ) @@ -139,7 +138,7 @@ async def _main(cfg: DictConfig): if upload_session_episodes_only: _LOG.info("Uploading session episode zips!") s3_uris = zip_and_upload_episodes( - get_identifier(address_eoa), + address_eoa, checkpoint_dir, _DEFAULT_EPISODES_S3_BUCKET, episode_runner.evaluate_dirs, @@ -147,7 +146,7 @@ async def _main(cfg: DictConfig): else: _LOG.info("Uploading all episode zips!") s3_uris = zip_and_upload_all_episodes( - get_identifier(address_eoa), + address_eoa, checkpoint_dir, _DEFAULT_EPISODES_S3_BUCKET, ) @@ -171,8 +170,9 @@ async def _main(cfg: DictConfig): is_telemetry_enabled = not telemetry.is_telemetry_disabled() git_ref = upload_to_huggingface( model_path=Path(model_dir), - user_id=get_identifier(address_eoa), + user_id=address_eoa, repo_id=hf_repo_id, + address_eoa=address_eoa, hf_token=hf_token, chain_metadata_dict={ "eoa": cfg.get("address_account"), diff --git a/src/blockassist/train.py b/src/blockassist/train.py index b7b7a07..218faf5 100644 --- a/src/blockassist/train.py +++ b/src/blockassist/train.py @@ -13,7 +13,6 @@ from blockassist.data import get_total_episodes from blockassist.globals import ( _DEFAULT_CHECKPOINT, - get_identifier, get_logger, ) from blockassist.goals.generator import BlockAssistGoalGenerator @@ -97,7 +96,7 @@ def after_training(self): duration_ms = int((self.end_time - self.start_time) * 1000) telemetry.push_telemetry_event_trained( duration_ms, - get_identifier(self.address_eoa), + self.address_eoa, get_total_episodes(self.checkpoint_dir), ) self.training_ended.set() From 8cc7da325a0c5bc5dc16f61b66e096e462c22f3b Mon Sep 17 00:00:00 2001 From: Eamonn Nugent <4699217+space55@users.noreply.github.com> Date: Thu, 2 Oct 2025 12:03:55 -0400 Subject: [PATCH 10/11] Add version --- run.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/run.py b/run.py index b999611..14e223b 100644 --- a/run.py +++ b/run.py @@ -7,6 +7,7 @@ import time from subprocess import Popen from typing import Dict, Optional +from importlib.metadata import version from rich.console import Console from rich.markdown import Markdown from rich.progress import ( @@ -21,6 +22,7 @@ import psutil import readchar + from daemon import PROCESSES, cleanup_processes, start_log_watcher TOTAL_TIME_PLAYED = 0 @@ -252,11 +254,12 @@ def run(): ██║ ██║███████║███████║██║███████║ ██║ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚══════╝ ╚═╝ -By Gensyn - """, +By Gensyn""", style=GENSYN_COLOR, ) + CONSOLE.print(f"Version {version('blockassist')}", style=GENSYN_COLOR) + if os.environ.get("HF_TOKEN") is None: logging.info("HF_TOKEN not found, prompting") CONSOLE.print( From ca767d425995c0fb861ea6bdd746cee9e5174e04 Mon Sep 17 00:00:00 2001 From: Eamonn Nugent <4699217+space55@users.noreply.github.com> Date: Thu, 2 Oct 2025 13:14:11 -0400 Subject: [PATCH 11/11] Fix version call --- pyproject.toml | 2 +- run.py | 17 +++++++++++++++-- scripts/get_version.py | 4 ++++ 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 scripts/get_version.py diff --git a/pyproject.toml b/pyproject.toml index 0b3cd55..fbbe75d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ name = "blockassist" description = "Distributed experiments built using the MBAG multiagent environment (https://github.com/cassidylaidlaw/minecraft-building-assistance-game)." readme = "README.md" requires-python = ">=3.10,<4.0" -version = "0.1.2" +version = "0.1.3" dependencies = [ "aiofiles", "boto3", diff --git a/run.py b/run.py index 14e223b..f5e3d67 100644 --- a/run.py +++ b/run.py @@ -2,6 +2,7 @@ import logging import os import signal +import subprocess import sys import threading import time @@ -38,6 +39,18 @@ GENSYN_COLOR = "bold magenta" +def get_version() -> str: + cmd = "./blockassist-venv/bin/python scripts/get_version.py" + process = Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if process.returncode != 0: + CONSOLE.print( + f"Error getting version: {stderr.decode().strip()}", style=ERROR_COLOR + ) + return "unknown" + return stdout.decode().strip() + + def create_logs_dir(clear_existing=True): if os.path.exists("logs") and clear_existing: CONSOLE.print("Clearing existing logs directory", style=LOG_COLOR) @@ -258,8 +271,6 @@ def run(): style=GENSYN_COLOR, ) - CONSOLE.print(f"Version {version('blockassist')}", style=GENSYN_COLOR) - if os.environ.get("HF_TOKEN") is None: logging.info("HF_TOKEN not found, prompting") CONSOLE.print( @@ -284,6 +295,8 @@ def run(): CONSOLE.print("Setting up virtualenv...", style=LOG_COLOR) setup_venv() + CONSOLE.print(f"BlockAssist version: {get_version()}", style=GENSYN_COLOR) + CONSOLE.print("Setting up Gradle...", style=LOG_COLOR) setup_gradle() diff --git a/scripts/get_version.py b/scripts/get_version.py new file mode 100644 index 0000000..9aed5a4 --- /dev/null +++ b/scripts/get_version.py @@ -0,0 +1,4 @@ +from importlib.metadata import version + + +print(version("blockassist"))