From f1c28543e02a8fc0aed4cc3de49833549f65ff40 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 29 Nov 2024 17:18:39 -0500 Subject: [PATCH 001/106] run-persistent --- docker/run-persistent.sh | 133 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 docker/run-persistent.sh diff --git a/docker/run-persistent.sh b/docker/run-persistent.sh new file mode 100644 index 0000000000..76176a39c7 --- /dev/null +++ b/docker/run-persistent.sh @@ -0,0 +1,133 @@ +#! /usr/bin/env bash +set -euo pipefail + +# Usage: ./run.sh +# Optional environment variables: FF_GPU_BACKEND, cuda_version, hip_version, ATTACH_GPUS, SHM_SIZE + +# Cd into directory holding this script +cd "${BASH_SOURCE[0]%/*}" + +# Parse input params +image=${1:-flexflow} +FF_GPU_BACKEND=${FF_GPU_BACKEND:-cuda} +cuda_version=${cuda_version:-"empty"} +hip_version=${hip_version:-"empty"} + +# Parameter controlling whether to attach GPUs to the Docker container +ATTACH_GPUS=${ATTACH_GPUS:-true} +gpu_arg="" +if $ATTACH_GPUS ; then gpu_arg="--gpus all" ; fi +FORWARD_STREAMLIT_PORT=${FORWARD_STREAMLIT_PORT:-true} +port_forward_arg="" +if $FORWARD_STREAMLIT_PORT ; then + port_forward_arg+="-p 8501:8501" +fi + + +# Amount of shared memory to give the Docker container access to +# If you get a Bus Error, increase this value. If you don't have enough memory +# on your machine, decrease this value. +SHM_SIZE=${SHM_SIZE:-8192m} + +# Check docker image name +if [[ "$image" != @(flexflow-environment|flexflow) ]]; then + echo "Error, image name ${image} is invalid. Choose between 'flexflow-environment', 'flexflow'." + exit 1 +fi + +# Check GPU backend +if [[ "${FF_GPU_BACKEND}" != @(cuda|hip_cuda|hip_rocm|intel) ]]; then + echo "Error, value of FF_GPU_BACKEND (${FF_GPU_BACKEND}) is invalid. Pick between 'cuda', 'hip_cuda', 'hip_rocm' or 'intel'." + exit 1 +elif [[ "${FF_GPU_BACKEND}" != "cuda" ]]; then + echo "Running $image docker image with gpu backend: ${FF_GPU_BACKEND}" +else + echo "Running $image docker image with default GPU backend: cuda" +fi + +# gpu backend version suffix for the docker image. +gpu_backend_version="" + +if [[ "${FF_GPU_BACKEND}" == "cuda" || "${FF_GPU_BACKEND}" == "hip_cuda" ]]; then + # Autodetect cuda version if not specified + if [[ $cuda_version == "empty" ]]; then + # shellcheck disable=SC2015 + cuda_version=$(command -v nvcc >/dev/null 2>&1 && nvcc --version | grep "release" | awk '{print $NF}' || true) + # Change cuda_version eg. V11.7.99 to 11.7 + cuda_version=${cuda_version:1:4} + if [[ -z "$cuda_version" ]]; then + echo "Could not detect CUDA version. Please specify one manually by setting the 'cuda_version' env." + exit 1 + fi + fi + # Check that CUDA version is supported + if [[ "$cuda_version" != @(11.1|11.2|11.3|11.4|11.5|11.6|11.7|11.8|12.0|12.1|12.2|12.3|12.4|12.5|12.6|12.7|12.8|12.9) ]]; then + echo "cuda_version is not supported, please choose among {11.1|11.2|11.3|11.4|11.5|11.6|11.7|11.8|12.0|12.1|12.2}" + exit 1 + fi + # Use CUDA 12.2 for all versions greater or equal to 12.2 for now + if [[ "$cuda_version" == @(12.3|12.4|12.5|12.6|12.7|12.8|12.9) ]]; then + cuda_version=12.2 + fi + # Set cuda version suffix to docker image name + echo "Running $image docker image with CUDA $cuda_version" + gpu_backend_version="-${cuda_version}" +fi + +if [[ "${FF_GPU_BACKEND}" == "hip_rocm" || "${FF_GPU_BACKEND}" == "hip_cuda" ]]; then + # Autodetect HIP version if not specified + if [[ $hip_version == "empty" ]]; then + # shellcheck disable=SC2015 + hip_version=$(command -v hipcc >/dev/null 2>&1 && hipcc --version | grep "HIP version:" | awk '{print $NF}' || true) + # Change hip_version eg. 5.6.31061-8c743ae5d to 5.6 + hip_version=${hip_version:0:3} + if [[ -z "$hip_version" ]]; then + echo "Could not detect HIP version. Please specify one manually by setting the 'hip_version' env." + exit 1 + fi + fi + # Check that HIP version is supported + if [[ "$hip_version" != @(5.3|5.4|5.5|5.6) ]]; then + echo "hip_version is not supported, please choose among {5.3, 5.4, 5.5, 5.6}" + exit 1 + fi + echo "Running $image docker image with HIP $hip_version" + if [[ "${FF_GPU_BACKEND}" == "hip_rocm" ]]; then + gpu_backend_version="-${hip_version}" + fi +fi + +# Check that image exists, if fails, print the default error message. +if [[ "$(docker images -q "${image}-${FF_GPU_BACKEND}${gpu_backend_version}":latest 2> /dev/null)" == "" ]]; then + echo "Error, ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest does not exist!" + if [[ "${FF_GPU_BACKEND}" == "cuda" ]]; then + echo "" + echo "To download the docker image, run:" + echo " FF_GPU_BACKEND=${FF_GPU_BACKEND} cuda_version=${cuda_version} $(pwd)/pull.sh $image" + echo "To build the docker image from source, run:" + echo " FF_GPU_BACKEND=${FF_GPU_BACKEND} cuda_version=${cuda_version} $(pwd)/build.sh $image" + echo "" + elif [[ "${FF_GPU_BACKEND}" == "hip_rocm" ]]; then + echo "" + echo "To download the docker image, run:" + echo " FF_GPU_BACKEND=${FF_GPU_BACKEND} hip_version=${hip_version} $(pwd)/pull.sh $image" + echo "To build the docker image from source, run:" + echo " FF_GPU_BACKEND=${FF_GPU_BACKEND} hip_version=${hip_version} $(pwd)/build.sh $image" + echo "" + fi + exit 1 +fi + +hf_token_volume="" +hf_token_path="$HOME/.cache/huggingface/token" +if [ -f "$hf_token_path" ]; then + # If the token exists, add the volume mount to the Docker command + hf_token_volume+="-v $hf_token_path:/root/.cache/huggingface/token" +fi + +ssh_key_volume="" +ssh_key_path="$HOME/.ssh/id_rsa" +if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then + ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" +fi +eval docker run -v my-volume:/home -it "$gpu_arg" "--shm-size=${SHM_SIZE}" "--cap-add=SYS_PTRACE" "${ssh_key_volume}" "${hf_token_volume}" "${port_forward_arg}" "${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" From e9a278d15e27c9661a08658daf7d82691c9d9b58 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 29 Nov 2024 22:19:09 +0000 Subject: [PATCH 002/106] permission --- docker/run-persistent.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 docker/run-persistent.sh diff --git a/docker/run-persistent.sh b/docker/run-persistent.sh old mode 100644 new mode 100755 From bd89698f897b62efd1c9d4279d4828988a61d640 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 29 Nov 2024 17:25:20 -0500 Subject: [PATCH 003/106] docker_command --- docker/run-persistent.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/run-persistent.sh b/docker/run-persistent.sh index 76176a39c7..b284d9bc82 100644 --- a/docker/run-persistent.sh +++ b/docker/run-persistent.sh @@ -130,4 +130,6 @@ ssh_key_path="$HOME/.ssh/id_rsa" if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" fi -eval docker run -v my-volume:/home -it "$gpu_arg" "--shm-size=${SHM_SIZE}" "--cap-add=SYS_PTRACE" "${ssh_key_volume}" "${hf_token_volume}" "${port_forward_arg}" "${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" +docker_command = "docker run -v my-volume:/home -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" +echo "$docker_command" +eval "$docker_command" \ No newline at end of file From 16fcc3d32f27f2b4fc262e6b38e6dd90fa58eac9 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 29 Nov 2024 17:26:14 -0500 Subject: [PATCH 004/106] spacing --- docker/run-persistent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/run-persistent.sh b/docker/run-persistent.sh index b284d9bc82..c24b8768cf 100755 --- a/docker/run-persistent.sh +++ b/docker/run-persistent.sh @@ -130,6 +130,6 @@ ssh_key_path="$HOME/.ssh/id_rsa" if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" fi -docker_command = "docker run -v my-volume:/home -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" +docker_command="docker run -v my-volume:/home -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" echo "$docker_command" eval "$docker_command" \ No newline at end of file From 03576289555f5b6ee78baf312c3af072604f80ba Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 29 Nov 2024 17:30:11 -0500 Subject: [PATCH 005/106] include ff --- docker/run-persistent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/run-persistent.sh b/docker/run-persistent.sh index c24b8768cf..936df23561 100755 --- a/docker/run-persistent.sh +++ b/docker/run-persistent.sh @@ -130,6 +130,6 @@ ssh_key_path="$HOME/.ssh/id_rsa" if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" fi -docker_command="docker run -v my-volume:/home -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" +docker_command="docker run -v my-volume:/FlexFlow -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" echo "$docker_command" eval "$docker_command" \ No newline at end of file From 58de6335ab6cc7dc224d24a794df83e22078334b Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 29 Nov 2024 17:38:55 -0500 Subject: [PATCH 006/106] persist all --- docker/run-persistent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/run-persistent.sh b/docker/run-persistent.sh index 936df23561..47c898c513 100755 --- a/docker/run-persistent.sh +++ b/docker/run-persistent.sh @@ -130,6 +130,6 @@ ssh_key_path="$HOME/.ssh/id_rsa" if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" fi -docker_command="docker run -v my-volume:/FlexFlow -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" +docker_command="docker run -v my-volume:/ -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" echo "$docker_command" eval "$docker_command" \ No newline at end of file From 682bd5ce57f67518a891c27fa7d63fd1c1eb0f99 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 29 Nov 2024 17:40:24 -0500 Subject: [PATCH 007/106] home --- docker/run-persistent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/run-persistent.sh b/docker/run-persistent.sh index 47c898c513..c24b8768cf 100755 --- a/docker/run-persistent.sh +++ b/docker/run-persistent.sh @@ -130,6 +130,6 @@ ssh_key_path="$HOME/.ssh/id_rsa" if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" fi -docker_command="docker run -v my-volume:/ -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" +docker_command="docker run -v my-volume:/home -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" echo "$docker_command" eval "$docker_command" \ No newline at end of file From 41f781a805fbd027771cc41021f3a50fd4f791f7 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 29 Nov 2024 20:09:35 -0500 Subject: [PATCH 008/106] detailed try..catch --- inference/models/mixtral.h | 161 ++++++++++++++++++++++++++++--------- 1 file changed, 121 insertions(+), 40 deletions(-) diff --git a/inference/models/mixtral.h b/inference/models/mixtral.h index 5c9817e886..2040f965c1 100644 --- a/inference/models/mixtral.h +++ b/inference/models/mixtral.h @@ -32,52 +32,133 @@ class MIXTRAL { try { json model_config; config_file >> model_config; - num_hidden_layers = model_config["num_hidden_layers"]; - vocab_size = model_config["vocab_size"]; - num_attention_heads = model_config["num_attention_heads"]; - if (model_config.find("num_key_value_heads") != model_config.end()) { - num_key_value_heads = model_config["num_key_value_heads"]; - } else { - num_key_value_heads = num_attention_heads; + try { + num_hidden_layers = model_config.at("num_hidden_layers"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'num_hidden_layers': " << e.what() << std::endl; + assert(false); } - hidden_size = model_config["hidden_size"]; - rms_norm_eps = model_config["rms_norm_eps"]; - intermediate_size = model_config["intermediate_size"]; - rotary_embedding_meta.apply_rotary_embedding = true; - if (model_config.find("rope_theta") != model_config.end()) { - rotary_embedding_meta.rope_theta = model_config["rope_theta"]; - } else { - rotary_embedding_meta.rope_theta = 10000.0f; + + try { + vocab_size = model_config.at("vocab_size"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'vocab_size': " << e.what() << std::endl; + assert(false); + } + + try { + num_attention_heads = model_config.at("num_attention_heads"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'num_attention_heads': " << e.what() << std::endl; + assert(false); + } + + try { + if (model_config.find("num_key_value_heads") != model_config.end()) { + num_key_value_heads = model_config["num_key_value_heads"]; + } else { + num_key_value_heads = num_attention_heads; + } + } catch (json::exception const &e) { + std::cerr << "Error reading 'num_key_value_heads': " << e.what() << std::endl; + assert(false); + } + + try { + hidden_size = model_config.at("hidden_size"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'hidden_size': " << e.what() << std::endl; + assert(false); + } + + try { + rms_norm_eps = model_config.at("rms_norm_eps"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'rms_norm_eps': " << e.what() << std::endl; + assert(false); + } + + try { + intermediate_size = model_config.at("intermediate_size"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'intermediate_size': " << e.what() << std::endl; + assert(false); + } + + try { + rotary_embedding_meta.apply_rotary_embedding = true; + if (model_config.find("rope_theta") != model_config.end()) { + rotary_embedding_meta.rope_theta = model_config["rope_theta"]; + } else { + rotary_embedding_meta.rope_theta = 10000.0f; + } + } catch (json::exception const &e) { + std::cerr << "Error reading 'rope_theta': " << e.what() << std::endl; + assert(false); + } + + try { + if (model_config.find("scaling_factor") != model_config.end() && + !model_config["scaling_factor"].is_null()) { + rotary_embedding_meta.rope_type = model_config["scaling_factor"]["rope_type"]; + rotary_embedding_meta.factor = model_config["scaling_factor"]["factor"]; + rotary_embedding_meta.low_freq_factor = model_config["scaling_factor"]["low_freq_factor"]; + rotary_embedding_meta.high_freq_factor = model_config["scaling_factor"]["high_freq_factor"]; + rotary_embedding_meta.original_max_position_embeddings = + model_config["scaling_factor"]["original_max_position_embeddings"]; + } + } catch (json::exception const &e) { + std::cerr << "Error reading 'scaling_factor': " << e.what() << std::endl; + assert(false); } - if (model_config.find("scaling_factor") != model_config.end() && - !model_config["scaling_factor"].is_null()) { - rotary_embedding_meta.rope_type = - model_config["scaling_factor"]["rope_type"]; - rotary_embedding_meta.factor = - model_config["scaling_factor"]["factor"]; - rotary_embedding_meta.low_freq_factor = - model_config["scaling_factor"]["low_freq_factor"]; - rotary_embedding_meta.high_freq_factor = - model_config["scaling_factor"]["high_freq_factor"]; - rotary_embedding_meta.original_max_position_embeddings = - model_config["scaling_factor"] - ["original_max_position_embeddings"]; - } - num_experts_per_tok = model_config["num_experts_per_tok"]; - num_local_experts = model_config["num_local_experts"]; - output_router_logits = model_config["output_router_logits"]; - router_aux_loss_coef = model_config["router_aux_loss_coef"]; - sliding_window = model_config["sliding_window"]; - tie_word_embeddings = model_config["tie_word_embeddings"]; + + try { + num_experts_per_tok = model_config.at("num_experts_per_tok"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'num_experts_per_tok': " << e.what() << std::endl; + assert(false); + } + + try { + num_local_experts = model_config.at("num_local_experts"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'num_local_experts': " << e.what() << std::endl; + assert(false); + } + + try { + output_router_logits = model_config.at("output_router_logits"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'output_router_logits': " << e.what() << std::endl; + assert(false); + } + + try { + router_aux_loss_coef = model_config.at("router_aux_loss_coef"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'router_aux_loss_coef': " << e.what() << std::endl; + assert(false); + } + + try { + sliding_window = model_config.at("sliding_window"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'sliding_window': " << e.what() << std::endl; + assert(false); + } + + try { + tie_word_embeddings = model_config.at("tie_word_embeddings"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'tie_word_embeddings': " << e.what() << std::endl; + assert(false); + } + } catch (json::exception const &e) { std::cerr << "Error parsing JSON file: " << e.what() << std::endl; assert(false); } - } else { - std::cerr << "Error opening JSON file " << model_config_file_path - << std::endl; - assert(false); - } + max_beam_width = BeamSearchBatchConfig::MAX_BEAM_WIDTH; max_beam_depth = BeamSearchBatchConfig::MAX_BEAM_DEPTH; } From feb859c3e9037d656bc2537737756c5ab91fd07a Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 29 Nov 2024 20:19:28 -0500 Subject: [PATCH 009/106] try to revert but errors --- inference/models/mixtral.h | 161 +++++++++---------------------------- 1 file changed, 40 insertions(+), 121 deletions(-) diff --git a/inference/models/mixtral.h b/inference/models/mixtral.h index 2040f965c1..5c9817e886 100644 --- a/inference/models/mixtral.h +++ b/inference/models/mixtral.h @@ -32,133 +32,52 @@ class MIXTRAL { try { json model_config; config_file >> model_config; - try { - num_hidden_layers = model_config.at("num_hidden_layers"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'num_hidden_layers': " << e.what() << std::endl; - assert(false); + num_hidden_layers = model_config["num_hidden_layers"]; + vocab_size = model_config["vocab_size"]; + num_attention_heads = model_config["num_attention_heads"]; + if (model_config.find("num_key_value_heads") != model_config.end()) { + num_key_value_heads = model_config["num_key_value_heads"]; + } else { + num_key_value_heads = num_attention_heads; } - - try { - vocab_size = model_config.at("vocab_size"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'vocab_size': " << e.what() << std::endl; - assert(false); - } - - try { - num_attention_heads = model_config.at("num_attention_heads"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'num_attention_heads': " << e.what() << std::endl; - assert(false); - } - - try { - if (model_config.find("num_key_value_heads") != model_config.end()) { - num_key_value_heads = model_config["num_key_value_heads"]; - } else { - num_key_value_heads = num_attention_heads; - } - } catch (json::exception const &e) { - std::cerr << "Error reading 'num_key_value_heads': " << e.what() << std::endl; - assert(false); - } - - try { - hidden_size = model_config.at("hidden_size"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'hidden_size': " << e.what() << std::endl; - assert(false); - } - - try { - rms_norm_eps = model_config.at("rms_norm_eps"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'rms_norm_eps': " << e.what() << std::endl; - assert(false); - } - - try { - intermediate_size = model_config.at("intermediate_size"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'intermediate_size': " << e.what() << std::endl; - assert(false); - } - - try { - rotary_embedding_meta.apply_rotary_embedding = true; - if (model_config.find("rope_theta") != model_config.end()) { - rotary_embedding_meta.rope_theta = model_config["rope_theta"]; - } else { - rotary_embedding_meta.rope_theta = 10000.0f; - } - } catch (json::exception const &e) { - std::cerr << "Error reading 'rope_theta': " << e.what() << std::endl; - assert(false); - } - - try { - if (model_config.find("scaling_factor") != model_config.end() && - !model_config["scaling_factor"].is_null()) { - rotary_embedding_meta.rope_type = model_config["scaling_factor"]["rope_type"]; - rotary_embedding_meta.factor = model_config["scaling_factor"]["factor"]; - rotary_embedding_meta.low_freq_factor = model_config["scaling_factor"]["low_freq_factor"]; - rotary_embedding_meta.high_freq_factor = model_config["scaling_factor"]["high_freq_factor"]; - rotary_embedding_meta.original_max_position_embeddings = - model_config["scaling_factor"]["original_max_position_embeddings"]; - } - } catch (json::exception const &e) { - std::cerr << "Error reading 'scaling_factor': " << e.what() << std::endl; - assert(false); + hidden_size = model_config["hidden_size"]; + rms_norm_eps = model_config["rms_norm_eps"]; + intermediate_size = model_config["intermediate_size"]; + rotary_embedding_meta.apply_rotary_embedding = true; + if (model_config.find("rope_theta") != model_config.end()) { + rotary_embedding_meta.rope_theta = model_config["rope_theta"]; + } else { + rotary_embedding_meta.rope_theta = 10000.0f; } - - try { - num_experts_per_tok = model_config.at("num_experts_per_tok"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'num_experts_per_tok': " << e.what() << std::endl; - assert(false); - } - - try { - num_local_experts = model_config.at("num_local_experts"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'num_local_experts': " << e.what() << std::endl; - assert(false); - } - - try { - output_router_logits = model_config.at("output_router_logits"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'output_router_logits': " << e.what() << std::endl; - assert(false); - } - - try { - router_aux_loss_coef = model_config.at("router_aux_loss_coef"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'router_aux_loss_coef': " << e.what() << std::endl; - assert(false); - } - - try { - sliding_window = model_config.at("sliding_window"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'sliding_window': " << e.what() << std::endl; - assert(false); - } - - try { - tie_word_embeddings = model_config.at("tie_word_embeddings"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'tie_word_embeddings': " << e.what() << std::endl; - assert(false); - } - + if (model_config.find("scaling_factor") != model_config.end() && + !model_config["scaling_factor"].is_null()) { + rotary_embedding_meta.rope_type = + model_config["scaling_factor"]["rope_type"]; + rotary_embedding_meta.factor = + model_config["scaling_factor"]["factor"]; + rotary_embedding_meta.low_freq_factor = + model_config["scaling_factor"]["low_freq_factor"]; + rotary_embedding_meta.high_freq_factor = + model_config["scaling_factor"]["high_freq_factor"]; + rotary_embedding_meta.original_max_position_embeddings = + model_config["scaling_factor"] + ["original_max_position_embeddings"]; + } + num_experts_per_tok = model_config["num_experts_per_tok"]; + num_local_experts = model_config["num_local_experts"]; + output_router_logits = model_config["output_router_logits"]; + router_aux_loss_coef = model_config["router_aux_loss_coef"]; + sliding_window = model_config["sliding_window"]; + tie_word_embeddings = model_config["tie_word_embeddings"]; } catch (json::exception const &e) { std::cerr << "Error parsing JSON file: " << e.what() << std::endl; assert(false); } - + } else { + std::cerr << "Error opening JSON file " << model_config_file_path + << std::endl; + assert(false); + } max_beam_width = BeamSearchBatchConfig::MAX_BEAM_WIDTH; max_beam_depth = BeamSearchBatchConfig::MAX_BEAM_DEPTH; } From e09ec2e0b1ac676da062365be91b104cb1b3a9d0 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 29 Nov 2024 20:31:40 -0500 Subject: [PATCH 010/106] add back else --- inference/models/mixtral.h | 155 ++++++++++++++++++++++++++++--------- 1 file changed, 120 insertions(+), 35 deletions(-) diff --git a/inference/models/mixtral.h b/inference/models/mixtral.h index 5c9817e886..87635203c9 100644 --- a/inference/models/mixtral.h +++ b/inference/models/mixtral.h @@ -32,43 +32,128 @@ class MIXTRAL { try { json model_config; config_file >> model_config; - num_hidden_layers = model_config["num_hidden_layers"]; - vocab_size = model_config["vocab_size"]; - num_attention_heads = model_config["num_attention_heads"]; - if (model_config.find("num_key_value_heads") != model_config.end()) { - num_key_value_heads = model_config["num_key_value_heads"]; - } else { - num_key_value_heads = num_attention_heads; + try { + num_hidden_layers = model_config.at("num_hidden_layers"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'num_hidden_layers': " << e.what() << std::endl; + assert(false); } - hidden_size = model_config["hidden_size"]; - rms_norm_eps = model_config["rms_norm_eps"]; - intermediate_size = model_config["intermediate_size"]; - rotary_embedding_meta.apply_rotary_embedding = true; - if (model_config.find("rope_theta") != model_config.end()) { - rotary_embedding_meta.rope_theta = model_config["rope_theta"]; - } else { - rotary_embedding_meta.rope_theta = 10000.0f; + + try { + vocab_size = model_config.at("vocab_size"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'vocab_size': " << e.what() << std::endl; + assert(false); + } + + try { + num_attention_heads = model_config.at("num_attention_heads"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'num_attention_heads': " << e.what() << std::endl; + assert(false); + } + + try { + if (model_config.find("num_key_value_heads") != model_config.end()) { + num_key_value_heads = model_config["num_key_value_heads"]; + } else { + num_key_value_heads = num_attention_heads; + } + } catch (json::exception const &e) { + std::cerr << "Error reading 'num_key_value_heads': " << e.what() << std::endl; + assert(false); + } + + try { + hidden_size = model_config.at("hidden_size"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'hidden_size': " << e.what() << std::endl; + assert(false); + } + + try { + rms_norm_eps = model_config.at("rms_norm_eps"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'rms_norm_eps': " << e.what() << std::endl; + assert(false); + } + + try { + intermediate_size = model_config.at("intermediate_size"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'intermediate_size': " << e.what() << std::endl; + assert(false); + } + + try { + rotary_embedding_meta.apply_rotary_embedding = true; + if (model_config.find("rope_theta") != model_config.end()) { + rotary_embedding_meta.rope_theta = model_config["rope_theta"]; + } else { + rotary_embedding_meta.rope_theta = 10000.0f; + } + } catch (json::exception const &e) { + std::cerr << "Error reading 'rope_theta': " << e.what() << std::endl; + assert(false); + } + + try { + if (model_config.find("scaling_factor") != model_config.end() && + !model_config["scaling_factor"].is_null()) { + rotary_embedding_meta.rope_type = model_config["scaling_factor"]["rope_type"]; + rotary_embedding_meta.factor = model_config["scaling_factor"]["factor"]; + rotary_embedding_meta.low_freq_factor = model_config["scaling_factor"]["low_freq_factor"]; + rotary_embedding_meta.high_freq_factor = model_config["scaling_factor"]["high_freq_factor"]; + rotary_embedding_meta.original_max_position_embeddings = + model_config["scaling_factor"]["original_max_position_embeddings"]; + } + } catch (json::exception const &e) { + std::cerr << "Error reading 'scaling_factor': " << e.what() << std::endl; + assert(false); + } + + try { + num_experts_per_tok = model_config.at("num_experts_per_tok"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'num_experts_per_tok': " << e.what() << std::endl; + assert(false); + } + + try { + num_local_experts = model_config.at("num_local_experts"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'num_local_experts': " << e.what() << std::endl; + assert(false); + } + + try { + output_router_logits = model_config.at("output_router_logits"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'output_router_logits': " << e.what() << std::endl; + assert(false); } - if (model_config.find("scaling_factor") != model_config.end() && - !model_config["scaling_factor"].is_null()) { - rotary_embedding_meta.rope_type = - model_config["scaling_factor"]["rope_type"]; - rotary_embedding_meta.factor = - model_config["scaling_factor"]["factor"]; - rotary_embedding_meta.low_freq_factor = - model_config["scaling_factor"]["low_freq_factor"]; - rotary_embedding_meta.high_freq_factor = - model_config["scaling_factor"]["high_freq_factor"]; - rotary_embedding_meta.original_max_position_embeddings = - model_config["scaling_factor"] - ["original_max_position_embeddings"]; - } - num_experts_per_tok = model_config["num_experts_per_tok"]; - num_local_experts = model_config["num_local_experts"]; - output_router_logits = model_config["output_router_logits"]; - router_aux_loss_coef = model_config["router_aux_loss_coef"]; - sliding_window = model_config["sliding_window"]; - tie_word_embeddings = model_config["tie_word_embeddings"]; + + try { + router_aux_loss_coef = model_config.at("router_aux_loss_coef"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'router_aux_loss_coef': " << e.what() << std::endl; + assert(false); + } + + try { + sliding_window = model_config.at("sliding_window"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'sliding_window': " << e.what() << std::endl; + assert(false); + } + + try { + tie_word_embeddings = model_config.at("tie_word_embeddings"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'tie_word_embeddings': " << e.what() << std::endl; + assert(false); + } + } catch (json::exception const &e) { std::cerr << "Error parsing JSON file: " << e.what() << std::endl; assert(false); From bdcfc94b418e4f5933b3d66c416c56983c0b6584 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 29 Nov 2024 21:13:51 -0500 Subject: [PATCH 011/106] echo docker command --- docker/run-bind.sh | 135 +++++++++++++++++++++++++++++++++++++++++++++ docker/run.sh | 5 +- 2 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 docker/run-bind.sh diff --git a/docker/run-bind.sh b/docker/run-bind.sh new file mode 100644 index 0000000000..c24b8768cf --- /dev/null +++ b/docker/run-bind.sh @@ -0,0 +1,135 @@ +#! /usr/bin/env bash +set -euo pipefail + +# Usage: ./run.sh +# Optional environment variables: FF_GPU_BACKEND, cuda_version, hip_version, ATTACH_GPUS, SHM_SIZE + +# Cd into directory holding this script +cd "${BASH_SOURCE[0]%/*}" + +# Parse input params +image=${1:-flexflow} +FF_GPU_BACKEND=${FF_GPU_BACKEND:-cuda} +cuda_version=${cuda_version:-"empty"} +hip_version=${hip_version:-"empty"} + +# Parameter controlling whether to attach GPUs to the Docker container +ATTACH_GPUS=${ATTACH_GPUS:-true} +gpu_arg="" +if $ATTACH_GPUS ; then gpu_arg="--gpus all" ; fi +FORWARD_STREAMLIT_PORT=${FORWARD_STREAMLIT_PORT:-true} +port_forward_arg="" +if $FORWARD_STREAMLIT_PORT ; then + port_forward_arg+="-p 8501:8501" +fi + + +# Amount of shared memory to give the Docker container access to +# If you get a Bus Error, increase this value. If you don't have enough memory +# on your machine, decrease this value. +SHM_SIZE=${SHM_SIZE:-8192m} + +# Check docker image name +if [[ "$image" != @(flexflow-environment|flexflow) ]]; then + echo "Error, image name ${image} is invalid. Choose between 'flexflow-environment', 'flexflow'." + exit 1 +fi + +# Check GPU backend +if [[ "${FF_GPU_BACKEND}" != @(cuda|hip_cuda|hip_rocm|intel) ]]; then + echo "Error, value of FF_GPU_BACKEND (${FF_GPU_BACKEND}) is invalid. Pick between 'cuda', 'hip_cuda', 'hip_rocm' or 'intel'." + exit 1 +elif [[ "${FF_GPU_BACKEND}" != "cuda" ]]; then + echo "Running $image docker image with gpu backend: ${FF_GPU_BACKEND}" +else + echo "Running $image docker image with default GPU backend: cuda" +fi + +# gpu backend version suffix for the docker image. +gpu_backend_version="" + +if [[ "${FF_GPU_BACKEND}" == "cuda" || "${FF_GPU_BACKEND}" == "hip_cuda" ]]; then + # Autodetect cuda version if not specified + if [[ $cuda_version == "empty" ]]; then + # shellcheck disable=SC2015 + cuda_version=$(command -v nvcc >/dev/null 2>&1 && nvcc --version | grep "release" | awk '{print $NF}' || true) + # Change cuda_version eg. V11.7.99 to 11.7 + cuda_version=${cuda_version:1:4} + if [[ -z "$cuda_version" ]]; then + echo "Could not detect CUDA version. Please specify one manually by setting the 'cuda_version' env." + exit 1 + fi + fi + # Check that CUDA version is supported + if [[ "$cuda_version" != @(11.1|11.2|11.3|11.4|11.5|11.6|11.7|11.8|12.0|12.1|12.2|12.3|12.4|12.5|12.6|12.7|12.8|12.9) ]]; then + echo "cuda_version is not supported, please choose among {11.1|11.2|11.3|11.4|11.5|11.6|11.7|11.8|12.0|12.1|12.2}" + exit 1 + fi + # Use CUDA 12.2 for all versions greater or equal to 12.2 for now + if [[ "$cuda_version" == @(12.3|12.4|12.5|12.6|12.7|12.8|12.9) ]]; then + cuda_version=12.2 + fi + # Set cuda version suffix to docker image name + echo "Running $image docker image with CUDA $cuda_version" + gpu_backend_version="-${cuda_version}" +fi + +if [[ "${FF_GPU_BACKEND}" == "hip_rocm" || "${FF_GPU_BACKEND}" == "hip_cuda" ]]; then + # Autodetect HIP version if not specified + if [[ $hip_version == "empty" ]]; then + # shellcheck disable=SC2015 + hip_version=$(command -v hipcc >/dev/null 2>&1 && hipcc --version | grep "HIP version:" | awk '{print $NF}' || true) + # Change hip_version eg. 5.6.31061-8c743ae5d to 5.6 + hip_version=${hip_version:0:3} + if [[ -z "$hip_version" ]]; then + echo "Could not detect HIP version. Please specify one manually by setting the 'hip_version' env." + exit 1 + fi + fi + # Check that HIP version is supported + if [[ "$hip_version" != @(5.3|5.4|5.5|5.6) ]]; then + echo "hip_version is not supported, please choose among {5.3, 5.4, 5.5, 5.6}" + exit 1 + fi + echo "Running $image docker image with HIP $hip_version" + if [[ "${FF_GPU_BACKEND}" == "hip_rocm" ]]; then + gpu_backend_version="-${hip_version}" + fi +fi + +# Check that image exists, if fails, print the default error message. +if [[ "$(docker images -q "${image}-${FF_GPU_BACKEND}${gpu_backend_version}":latest 2> /dev/null)" == "" ]]; then + echo "Error, ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest does not exist!" + if [[ "${FF_GPU_BACKEND}" == "cuda" ]]; then + echo "" + echo "To download the docker image, run:" + echo " FF_GPU_BACKEND=${FF_GPU_BACKEND} cuda_version=${cuda_version} $(pwd)/pull.sh $image" + echo "To build the docker image from source, run:" + echo " FF_GPU_BACKEND=${FF_GPU_BACKEND} cuda_version=${cuda_version} $(pwd)/build.sh $image" + echo "" + elif [[ "${FF_GPU_BACKEND}" == "hip_rocm" ]]; then + echo "" + echo "To download the docker image, run:" + echo " FF_GPU_BACKEND=${FF_GPU_BACKEND} hip_version=${hip_version} $(pwd)/pull.sh $image" + echo "To build the docker image from source, run:" + echo " FF_GPU_BACKEND=${FF_GPU_BACKEND} hip_version=${hip_version} $(pwd)/build.sh $image" + echo "" + fi + exit 1 +fi + +hf_token_volume="" +hf_token_path="$HOME/.cache/huggingface/token" +if [ -f "$hf_token_path" ]; then + # If the token exists, add the volume mount to the Docker command + hf_token_volume+="-v $hf_token_path:/root/.cache/huggingface/token" +fi + +ssh_key_volume="" +ssh_key_path="$HOME/.ssh/id_rsa" +if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then + ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" +fi +docker_command="docker run -v my-volume:/home -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" +echo "$docker_command" +eval "$docker_command" \ No newline at end of file diff --git a/docker/run.sh b/docker/run.sh index 62d7468a00..99c5db9ccc 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -130,4 +130,7 @@ ssh_key_path="$HOME/.ssh/id_rsa" if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" fi -eval docker run -it "$gpu_arg" "--shm-size=${SHM_SIZE}" "--cap-add=SYS_PTRACE" "${ssh_key_volume}" "${hf_token_volume}" "${port_forward_arg}" "${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" + +docker_command="docker run -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" +echo "$docker_command" +eval "$docker_command" From 1c8805827a83ebda4bd7cbb44aad4cb9c8d355d7 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 29 Nov 2024 22:09:05 -0500 Subject: [PATCH 012/106] no sliding window --- inference/models/mixtral.h | 16 ++++++++-------- python/flexflow/serve/models/mixtral.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/inference/models/mixtral.h b/inference/models/mixtral.h index 87635203c9..416f116441 100644 --- a/inference/models/mixtral.h +++ b/inference/models/mixtral.h @@ -140,12 +140,12 @@ class MIXTRAL { assert(false); } - try { - sliding_window = model_config.at("sliding_window"); - } catch (json::exception const &e) { - std::cerr << "Error reading 'sliding_window': " << e.what() << std::endl; - assert(false); - } +// try { // TODO sliding_window is null in the JSON config file of mistralai/Mixtral-8x7B-v0.1 +// sliding_window = model_config.at("sliding_window"); +// } catch (json::exception const &e) { +// std::cerr << "Error reading 'sliding_window': " << e.what() << std::endl; +// assert(false); +// } try { tie_word_embeddings = model_config.at("tie_word_embeddings"); @@ -187,7 +187,7 @@ class MIXTRAL { std::cout << "\trope_theta: " << rope_theta << std::endl; std::cout << "\trouter_aux_loss_coef: " << router_aux_loss_coef << std::endl; - std::cout << "\tsliding_window: " << sliding_window << std::endl; +// std::cout << "\tsliding_window: " << sliding_window << std::endl; // TODO std::cout << "\ttie_word_embeddings: " << tie_word_embeddings << std::endl; std::cout << "\tvocab_size: " << vocab_size << std::endl; @@ -203,7 +203,7 @@ class MIXTRAL { float rms_norm_eps; float rope_theta; float router_aux_loss_coef; - int sliding_window; +// int sliding_window; // TODO bool tie_word_embeddings; int vocab_size; int max_beam_width, max_beam_depth; diff --git a/python/flexflow/serve/models/mixtral.py b/python/flexflow/serve/models/mixtral.py index e00eaa07f7..205b430a78 100644 --- a/python/flexflow/serve/models/mixtral.py +++ b/python/flexflow/serve/models/mixtral.py @@ -34,7 +34,7 @@ def __init__(self, hf_config): self.rms_norm_eps = hf_config.rms_norm_eps self.rope_theta = hf_config.rope_theta self.router_aux_loss_coef = hf_config.router_aux_loss_coef - self.sliding_window = hf_config.sliding_window + # self.sliding_window = hf_config.sliding_window // TODO self.tie_word_embeddings = hf_config.tie_word_embeddings self.vocab_size = hf_config.vocab_size From 14f5820dde0812cc8248e60dddb2c5658a61a274 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 29 Nov 2024 22:16:08 -0500 Subject: [PATCH 013/106] read max_position_embeddings --- inference/models/mixtral.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/inference/models/mixtral.h b/inference/models/mixtral.h index 416f116441..3621ebfa2e 100644 --- a/inference/models/mixtral.h +++ b/inference/models/mixtral.h @@ -32,6 +32,14 @@ class MIXTRAL { try { json model_config; config_file >> model_config; + + try { + max_position_embeddings = model_config.at("max_position_embeddings"); + } catch (json::exception const &e) { + std::cerr << "Error reading 'max_position_embeddings': " << e.what() << std::endl; + assert(false); + } + try { num_hidden_layers = model_config.at("num_hidden_layers"); } catch (json::exception const &e) { From 670b8cbfc7d928150271a49a3422d8618559fe47 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 30 Nov 2024 12:34:01 -0500 Subject: [PATCH 014/106] run-persistent --- docker/run-persistent.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docker/run-persistent.sh b/docker/run-persistent.sh index c24b8768cf..547fe77cdc 100755 --- a/docker/run-persistent.sh +++ b/docker/run-persistent.sh @@ -118,18 +118,17 @@ if [[ "$(docker images -q "${image}-${FF_GPU_BACKEND}${gpu_backend_version}":lat exit 1 fi -hf_token_volume="" -hf_token_path="$HOME/.cache/huggingface/token" -if [ -f "$hf_token_path" ]; then - # If the token exists, add the volume mount to the Docker command - hf_token_volume+="-v $hf_token_path:/root/.cache/huggingface/token" -fi +#cache_volume="-v ${HOME}/.cache:/root/.cache" +#home_volume="-v ${HOME}/dockerhome:/home" +cache_volume="-v cache_volume:/root/.cache" +home_volume="-v home_volume:/home" ssh_key_volume="" ssh_key_path="$HOME/.ssh/id_rsa" if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" fi -docker_command="docker run -v my-volume:/home -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" + +docker_command="docker run -v my-volume:/home -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${cache_volume} ${home_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" echo "$docker_command" eval "$docker_command" \ No newline at end of file From 812deb007c1a385aee53c49afb5ab7f67597b679 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 30 Nov 2024 12:36:12 -0500 Subject: [PATCH 015/106] home once only --- .gitignore | 4 ++++ docker/run-persistent.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c1e22fcaba..4866ebbc90 100644 --- a/.gitignore +++ b/.gitignore @@ -196,3 +196,7 @@ tests/inference/python_test_configs/*.json core.* fine_grained_alignment_config.json + +# CLion +.idea/ +cmake-build-debug \ No newline at end of file diff --git a/docker/run-persistent.sh b/docker/run-persistent.sh index 547fe77cdc..162ba72795 100755 --- a/docker/run-persistent.sh +++ b/docker/run-persistent.sh @@ -129,6 +129,6 @@ if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" fi -docker_command="docker run -v my-volume:/home -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${cache_volume} ${home_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" +docker_command="docker run -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${cache_volume} ${home_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" echo "$docker_command" eval "$docker_command" \ No newline at end of file From 318d052a69de7e4d3fd4998cfa64d79b851b1f50 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Mon, 2 Dec 2024 10:05:55 -0500 Subject: [PATCH 016/106] config --- config/config.linux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.linux b/config/config.linux index aae7901494..cbd876c3e2 100755 --- a/config/config.linux +++ b/config/config.linux @@ -13,7 +13,7 @@ INSTALL_DIR=${INSTALL_DIR:-} # set build type -BUILD_TYPE=${BUILD_TYPE:-Release} +BUILD_TYPE=${BUILD_TYPE:-Debug} INFERENCE_TESTS=${INFERENCE_TESTS:-OFF} LIBTORCH_PATH=${LIBTORCH_PATH:-"$(realpath ../..)/libtorch"} From 17f162faa379027232651a4c447747b2b586656d Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Mon, 2 Dec 2024 10:42:18 -0500 Subject: [PATCH 017/106] with . --- inference/models/mixtral.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 9912011a2f..5dee24d2c2 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -291,7 +291,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, ff.sigmoid_silu_multi(w1, w3, DT_NONE, - std::string("layers_" + std::to_string(i) + + std::string("layers." + std::to_string(i) + "_block_sparse_moe_experts_" + std::to_string(expert_idx) + "ssm") .c_str()); @@ -306,7 +306,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers_" + std::to_string(i) + + std::string("layers." + std::to_string(i) + "_block_sparse_moe_experts_" + std::to_string(expert_idx) + "_w2") .c_str()); @@ -322,8 +322,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mlp_out = ff.aggregate(aggregate_inputs, mixtral_config.num_local_experts, 0.0f, - std::string("layers_" + std::to_string(i) + - "_block_sparse_moe_experts_aggregate") + std::string("layers." + std::to_string(i) + + ".block_sparse_moe_experts_aggregate") .c_str()); } // final normalization and linear From f5c3f7f72feed2e266025c6caeb3c7e3675ef1f9 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Tue, 3 Dec 2024 13:53:28 -0500 Subject: [PATCH 018/106] all _ instead of . --- inference/models/mixtral.cc | 34 ++++++++++++------------- python/flexflow/serve/models/mixtral.py | 1 + 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 5dee24d2c2..1c11b50cb2 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -76,7 +76,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.rms_norm_eps, mixtral_config.hidden_size, DT_NONE, - std::string("layers." + std::to_string(i) + ".input_layernorm") + std::string("layers_" + std::to_string(i) + ".input_layernorm") .c_str()); } else { ff.residual_rms_norm( @@ -87,7 +87,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.hidden_size, false, // inplace_residual DT_NONE, - std::string("layers." + std::to_string(i) + ".input_layernorm") + std::string("layers_" + std::to_string(i) + ".input_layernorm") .c_str()); token = token_att_norm[0]; att_norm = token_att_norm[1]; @@ -105,7 +105,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, // ? REG_MODE_NONE, // no regularization 0.0f, // no dropout - std::string("layers." + std::to_string(i) + ".self_attn.qkv_proj") + std::string("layers_" + std::to_string(i) + ".self_attn.qkv_proj") .c_str()); Tensor mha; @@ -127,7 +127,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 1.0f, /*scaling factor*/ true, /*qk_prod_scaling*/ false, /*position_bias*/ - std::string("layers." + std::to_string(i) + ".self_attn") + std::string("layers_" + std::to_string(i) + ".self_attn") .c_str() /*name*/ ); break; @@ -149,7 +149,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 1.0f, /*scaling factor*/ true, /*qk_prod_scaling*/ false, /*position_bias*/ - std::string("layers." + std::to_string(i) + ".self_attn") + std::string("layers_" + std::to_string(i) + ".self_attn") .c_str() /*name*/ ); break; @@ -171,7 +171,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 1.0f, /*scaling factor*/ true, /*qk_prod_scaling*/ false, /*position_bias*/ - std::string("layers." + std::to_string(i) + ".self_attn") + std::string("layers_" + std::to_string(i) + ".self_attn") .c_str() /*name*/ ); break; @@ -193,7 +193,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers." + std::to_string(i) + ".self_attn.o_proj") + std::string("layers_" + std::to_string(i) + ".self_attn.o_proj") .c_str()); @@ -207,7 +207,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.hidden_size, false, // inplace_residual DT_NONE, - std::string("layers." + std::to_string(i) + ".post_attention_layernorm") + std::string("layers_" + std::to_string(i) + ".post_attention_layernorm") .c_str()); token = token_ff_norm[0]; Tensor ff_norm = token_ff_norm[1]; @@ -224,13 +224,13 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers." + std::to_string(i) + "_block_sparse_moe_gate") + std::string("layers_" + std::to_string(i) + "_block_sparse_moe_gate") .c_str()); gate = ff.softmax( gate, 0, DT_NONE, - std::string("layers." + std::to_string(i) + "_block_sparse_moe_softmax") + std::string("layers_" + std::to_string(i) + "_block_sparse_moe_softmax") .c_str()); Tensor topk_out[2] = {nullptr, nullptr}; @@ -239,7 +239,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, topk_out, mixtral_config.num_experts_per_tok, false, - std::string("layers." + std::to_string(i) + "_block_sparse_moe_topk") + std::string("layers_" + std::to_string(i) + "_block_sparse_moe_topk") .c_str()); Tensor topk_values = topk_out[0]; Tensor topk_indices = topk_out[1]; @@ -251,7 +251,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, grouped_tokens, mixtral_config.num_local_experts, 0.0f, - std::string("layers." + std::to_string(i) + "_block_sparse_moe_groupby") + std::string("layers_" + std::to_string(i) + "_block_sparse_moe_groupby") .c_str()); Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; @@ -267,7 +267,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers." + std::to_string(i) + + std::string("layers_" + std::to_string(i) + "_block_sparse_moe_experts_" + std::to_string(expert_idx) + "_w1") .c_str()); @@ -282,7 +282,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers." + std::to_string(i) + + std::string("layers_" + std::to_string(i) + "_block_sparse_moe_experts_" + std::to_string(expert_idx) + "_w3") .c_str()); @@ -291,7 +291,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, ff.sigmoid_silu_multi(w1, w3, DT_NONE, - std::string("layers." + std::to_string(i) + + std::string("layers_" + std::to_string(i) + "_block_sparse_moe_experts_" + std::to_string(expert_idx) + "ssm") .c_str()); @@ -306,7 +306,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers." + std::to_string(i) + + std::string("layers_" + std::to_string(i) + "_block_sparse_moe_experts_" + std::to_string(expert_idx) + "_w2") .c_str()); @@ -322,7 +322,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mlp_out = ff.aggregate(aggregate_inputs, mixtral_config.num_local_experts, 0.0f, - std::string("layers." + std::to_string(i) + + std::string("layers_" + std::to_string(i) + ".block_sparse_moe_experts_aggregate") .c_str()); } diff --git a/python/flexflow/serve/models/mixtral.py b/python/flexflow/serve/models/mixtral.py index 205b430a78..cd758acdca 100644 --- a/python/flexflow/serve/models/mixtral.py +++ b/python/flexflow/serve/models/mixtral.py @@ -282,5 +282,6 @@ def build_model(self, max_tokens_per_batch): def convert_hf_model(model, dst_folder): os.makedirs(dst_folder, exist_ok=True) for name, params in model.named_parameters(): + print("name", name) name = name.replace(".", "_").replace("model_", "") params.detach().cpu().numpy().tofile(f"{dst_folder}/{name}") From 22e2a5885dee62be1e1bc02a4c357e70dc186c77 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Tue, 3 Dec 2024 13:58:35 -0500 Subject: [PATCH 019/106] _block --- inference/models/mixtral.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 1c11b50cb2..2235ce7030 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -323,7 +323,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.num_local_experts, 0.0f, std::string("layers_" + std::to_string(i) + - ".block_sparse_moe_experts_aggregate") + "_block_sparse_moe_experts_aggregate") .c_str()); } // final normalization and linear From 1cd424a4b3a013c84544e7bb423e3c106f4094e3 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Tue, 3 Dec 2024 14:36:59 -0500 Subject: [PATCH 020/106] try nullptr --- inference/models/mixtral.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 2235ce7030..2bdc159b33 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -322,9 +322,12 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mlp_out = ff.aggregate(aggregate_inputs, mixtral_config.num_local_experts, 0.0f, - std::string("layers_" + std::to_string(i) + - "_block_sparse_moe_experts_aggregate") - .c_str()); + nullptr, // TODO figure out name +// std::string("layers_" + std::to_string(i) + +// "_block_sparse_moe_experts_aggregate") +// .c_str() +// + ); } // final normalization and linear Tensor final_rms_norm_output[2] = {nullptr, nullptr}; From df0c59b00df5a53d4475a9887c707a80608f24c9 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Tue, 3 Dec 2024 14:40:21 -0500 Subject: [PATCH 021/106] no comma --- inference/models/mixtral.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 2bdc159b33..eee1ec18ef 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -322,7 +322,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mlp_out = ff.aggregate(aggregate_inputs, mixtral_config.num_local_experts, 0.0f, - nullptr, // TODO figure out name + nullptr // TODO figure out name // std::string("layers_" + std::to_string(i) + // "_block_sparse_moe_experts_aggregate") // .c_str() From 3598ee969ff5dcd1735cbf1774f85417d5214d1b Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Tue, 3 Dec 2024 15:37:28 -0500 Subject: [PATCH 022/106] debug --- inference/models/mixtral.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index eee1ec18ef..645bb5e227 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -318,7 +318,9 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[0] = topk_values; aggregate_inputs[1] = topk_indices; - aggregate_inputs[2] = aggregate_inputs[3] = nullptr; + aggregate_inputs[2] = 1; + aggregate_inputs[3] = 1; +// aggregate_inputs[2] = aggregate_inputs[3] = nullptr; mlp_out = ff.aggregate(aggregate_inputs, mixtral_config.num_local_experts, 0.0f, From 06259f7e599d16125d1fde1de35c5872ecde050b Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Tue, 3 Dec 2024 15:38:23 -0500 Subject: [PATCH 023/106] debug --- inference/models/mixtral.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 645bb5e227..7755481f99 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -318,8 +318,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[0] = topk_values; aggregate_inputs[1] = topk_indices; - aggregate_inputs[2] = 1; - aggregate_inputs[3] = 1; + aggregate_inputs[2] = topk_values; + aggregate_inputs[3] = topk_indices; // aggregate_inputs[2] = aggregate_inputs[3] = nullptr; mlp_out = ff.aggregate(aggregate_inputs, mixtral_config.num_local_experts, From c89a3277909027c3d22c09d66fd817f532a5822e Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Fri, 6 Dec 2024 17:03:42 -0500 Subject: [PATCH 024/106] revert aggregate --- inference/models/mixtral.cc | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 7755481f99..2235ce7030 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -318,18 +318,13 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[0] = topk_values; aggregate_inputs[1] = topk_indices; - aggregate_inputs[2] = topk_values; - aggregate_inputs[3] = topk_indices; -// aggregate_inputs[2] = aggregate_inputs[3] = nullptr; + aggregate_inputs[2] = aggregate_inputs[3] = nullptr; mlp_out = ff.aggregate(aggregate_inputs, mixtral_config.num_local_experts, 0.0f, - nullptr // TODO figure out name -// std::string("layers_" + std::to_string(i) + -// "_block_sparse_moe_experts_aggregate") -// .c_str() -// - ); + std::string("layers_" + std::to_string(i) + + "_block_sparse_moe_experts_aggregate") + .c_str()); } // final normalization and linear Tensor final_rms_norm_output[2] = {nullptr, nullptr}; From 71c5d699240ff6839357151af6aa929bcfd39735 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 11:11:14 -0500 Subject: [PATCH 025/106] one expert --- inference/models/mixtral.cc | 165 ++++++++++++------------------------ 1 file changed, 56 insertions(+), 109 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 2235ce7030..b4f0e3070a 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -213,118 +213,65 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor ff_norm = token_ff_norm[1]; // MoE - Tensor gate = ff.dense( - ff_norm, - mixtral_config.num_local_experts, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers_" + std::to_string(i) + "_block_sparse_moe_gate") - .c_str()); - gate = ff.softmax( - gate, - 0, - DT_NONE, - std::string("layers_" + std::to_string(i) + "_block_sparse_moe_softmax") - .c_str()); - Tensor topk_out[2] = {nullptr, nullptr}; - ff.top_k( - gate, - topk_out, - mixtral_config.num_experts_per_tok, - false, - std::string("layers_" + std::to_string(i) + "_block_sparse_moe_topk") - .c_str()); - Tensor topk_values = topk_out[0]; - Tensor topk_indices = topk_out[1]; - - Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; - ff.group_by( - ff_norm, - topk_indices, - grouped_tokens, - mixtral_config.num_local_experts, - 0.0f, - std::string("layers_" + std::to_string(i) + "_block_sparse_moe_groupby") - .c_str()); + Tensor w1 = ff.dense(ff_norm, + mixtral_config.intermediate_size, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers_" + std::to_string(i) + + "_block_sparse_moe_experts_" + + std::to_string(expert_idx) + "_w1") + .c_str()); + + Tensor w3 = ff.dense(ff_norm, + mixtral_config.intermediate_size, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers_" + std::to_string(i) + + "_block_sparse_moe_experts_0_w3") + .c_str()); + + Tensor multi = + ff.sigmoid_silu_multi(w1, + w3); +// DT_NONE, +// std::string("layers_" + std::to_string(i) + +// "_block_sparse_moe_experts_" + +// std::to_string(expert_idx) + "ssm") +// .c_str()); + + Tensor mlp_out = ff.dense(multi, + mixtral_config.hidden_size, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers_" + std::to_string(i) + + "_block_sparse_moe_experts_0_w2") + .c_str()); + + + + + - Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; - for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; - expert_idx++) { - Tensor w1 = ff.dense(grouped_tokens[expert_idx], - mixtral_config.intermediate_size, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers_" + std::to_string(i) + - "_block_sparse_moe_experts_" + - std::to_string(expert_idx) + "_w1") - .c_str()); - - Tensor w3 = ff.dense(grouped_tokens[expert_idx], - mixtral_config.intermediate_size, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers_" + std::to_string(i) + - "_block_sparse_moe_experts_" + - std::to_string(expert_idx) + "_w3") - .c_str()); - - Tensor multi = - ff.sigmoid_silu_multi(w1, - w3, - DT_NONE, - std::string("layers_" + std::to_string(i) + - "_block_sparse_moe_experts_" + - std::to_string(expert_idx) + "ssm") - .c_str()); - - Tensor w2 = ff.dense(multi, - mixtral_config.hidden_size, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers_" + std::to_string(i) + - "_block_sparse_moe_experts_" + - std::to_string(expert_idx) + "_w2") - .c_str()); - aggregate_inputs[4 + expert_idx] = w2; - } - Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); - topk_values = ff.divide(topk_values, topk_values_reduced); - - aggregate_inputs[0] = topk_values; - aggregate_inputs[1] = topk_indices; - aggregate_inputs[2] = aggregate_inputs[3] = nullptr; - mlp_out = ff.aggregate(aggregate_inputs, - mixtral_config.num_local_experts, - 0.0f, - std::string("layers_" + std::to_string(i) + - "_block_sparse_moe_experts_aggregate") - .c_str()); } // final normalization and linear Tensor final_rms_norm_output[2] = {nullptr, nullptr}; From cb1eaa8de00cd5579c4f6ab3dc9876a8a7b5f6f9 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 11:12:03 -0500 Subject: [PATCH 026/106] sync --- inference/models/mixtral.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index b4f0e3070a..27b31ab114 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -225,8 +225,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, REG_MODE_NONE, 0.0f, std::string("layers_" + std::to_string(i) + - "_block_sparse_moe_experts_" + - std::to_string(expert_idx) + "_w1") + "_block_sparse_moe_experts_0_w1") .c_str()); Tensor w3 = ff.dense(ff_norm, From 286a1fe54ed37eb4eb845242b2049174a8a8fde6 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 11:23:13 -0500 Subject: [PATCH 027/106] dont redefien mlpout --- inference/models/mixtral.cc | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 27b31ab114..a2a8d918ff 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -251,7 +251,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, // std::to_string(expert_idx) + "ssm") // .c_str()); - Tensor mlp_out = ff.dense(multi, + mlp_out = ff.dense(multi, mixtral_config.hidden_size, AC_MODE_NONE, false, @@ -265,12 +265,6 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, "_block_sparse_moe_experts_0_w2") .c_str()); - - - - - - } // final normalization and linear Tensor final_rms_norm_output[2] = {nullptr, nullptr}; From 8709035fdc46a75594c08a4ee21c24435eb1ed20 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 11:40:54 -0500 Subject: [PATCH 028/106] sync --- inference/models/llama.cc | 62 ++------------------------------------- 1 file changed, 2 insertions(+), 60 deletions(-) diff --git a/inference/models/llama.cc b/inference/models/llama.cc index 7b4a14b472..c3080e2082 100644 --- a/inference/models/llama.cc +++ b/inference/models/llama.cc @@ -109,50 +109,8 @@ void LLAMA::create_llama_model(FFModel &ff, Tensor mha; switch (mode) { - case BEAM_SEARCH_MODE: { - mha = ff.spec_inc_multiquery_self_attention( - qkv_proj, - llama_config.hidden_size, - llama_config.num_attention_heads, - llama_config.num_key_value_heads, - llama_config.hidden_size / llama_config.num_attention_heads, - llama_config.hidden_size / llama_config.num_attention_heads, - 0.0f, /*dropout*/ - false, /*add_zero_attn*/ - DT_NONE, /*data_type*/ - NULL, /*kernel_initializer*/ - llama_config.rotary_embedding_meta, - false, /*scaling query*/ - 1.0f, /*scaling factor*/ - true, /*qk_prod_scaling*/ - false, /*position_bias*/ - std::string("layers." + std::to_string(i) + ".self_attn") - .c_str() /*name*/ - ); - break; - } - case TREE_VERIFY_MODE: { - mha = ff.inc_multiquery_self_attention_verify( - qkv_proj, - llama_config.hidden_size, - llama_config.num_attention_heads, - llama_config.num_key_value_heads, - llama_config.hidden_size / llama_config.num_attention_heads, - llama_config.hidden_size / llama_config.num_attention_heads, - 0.0f, /*dropout*/ - false, /*add_zero_attn*/ - DT_NONE, /*data_type*/ - nullptr, /*kernel_initializer*/ - llama_config.rotary_embedding_meta, - false, /*scaling query*/ - 1.0f, /*scaling factor*/ - true, /*qk_prod_scaling*/ - false, /*position_bias*/ - std::string("layers." + std::to_string(i) + ".self_attn") - .c_str() /*name*/ - ); - break; - } + + case INC_DECODING_MODE: { mha = ff.inc_multiquery_self_attention( qkv_proj, @@ -275,13 +233,6 @@ void LLAMA::create_llama_model(FFModel &ff, "lm_head"); Tensor output; - if (mode == BEAM_SEARCH_MODE) { - Tensor softmax = ff.softmax(dense, -1); - // output = ff.beam_top_k(softmax, llama_config.max_beam_width, false); - // output = ff.argmax(softmax, /*beam_Search*/ true); - output = ff.arg_top_k(softmax, llama_config.max_beam_width, false, true); - // output = ff.top_k(softmax, ) - } else { // Tensor softmax = ff.softmax(dense, -1); if (generation_config.do_sample) { dense = ff.scalar_truediv(dense, generation_config.temperature, false); @@ -292,15 +243,6 @@ void LLAMA::create_llama_model(FFModel &ff, Tensor softmax = ff.softmax(dense, -1); output = ff.argmax(softmax, /*beam_Search*/ false); } - } - - // If PEFT is enabled, add LoRA layers - if (ff.config.enable_peft) { - // todo: add attention projections - std::vector target_modules = { - "gate_proj", "up_proj", "down_proj"}; - ff.add_lora_layers(target_modules); - } FileDataLoader *fileloader = new FileDataLoader( "", From 51f9701192472f0fd742691330626d63f9055ca5 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 11:50:58 -0500 Subject: [PATCH 029/106] sync --- inference/models/llama.cc | 2 - inference/models/mixtral.cc | 97 ++++++------------------------------- 2 files changed, 15 insertions(+), 84 deletions(-) diff --git a/inference/models/llama.cc b/inference/models/llama.cc index c3080e2082..157cd7c158 100644 --- a/inference/models/llama.cc +++ b/inference/models/llama.cc @@ -109,8 +109,6 @@ void LLAMA::create_llama_model(FFModel &ff, Tensor mha; switch (mode) { - - case INC_DECODING_MODE: { mha = ff.inc_multiquery_self_attention( qkv_proj, diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index a2a8d918ff..a082e6b7c8 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -26,13 +26,12 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, InferenceMode mode, GenerationConfig generation_config, bool use_full_precision) { + MixtralConfig mixtral_config(model_config_file_path); mixtral_config.print(); - if (ff.config.tensor_parallelism_degree > - mixtral_config.num_attention_heads || - mixtral_config.num_attention_heads % - ff.config.tensor_parallelism_degree != + if (ff.config.tensor_parallelism_degree > mixtral_config.num_attention_heads || + mixtral_config.num_attention_heads % ff.config.tensor_parallelism_degree != 0) { assert(false && "The number of attention heads is smaller, or it is not " "divisible by the tensor parallelism degree"); @@ -110,50 +109,6 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor mha; switch (mode) { - case BEAM_SEARCH_MODE: { - mha = ff.spec_inc_multiquery_self_attention( - qkv_proj, - mixtral_config.hidden_size, - mixtral_config.num_attention_heads, - mixtral_config.num_key_value_heads, - mixtral_config.hidden_size / mixtral_config.num_attention_heads, - mixtral_config.hidden_size / mixtral_config.num_attention_heads, - 0.0f, /*dropout*/ - false, /*add_zero_attn*/ - DT_NONE, /*data_type*/ - NULL, /*kernel_initializer*/ - mixtral_config.rotary_embedding_meta, - false, /*scaling query*/ - 1.0f, /*scaling factor*/ - true, /*qk_prod_scaling*/ - false, /*position_bias*/ - std::string("layers_" + std::to_string(i) + ".self_attn") - .c_str() /*name*/ - ); - break; - } - case TREE_VERIFY_MODE: { - mha = ff.inc_multiquery_self_attention_verify( - qkv_proj, - mixtral_config.hidden_size, - mixtral_config.num_attention_heads, - mixtral_config.num_key_value_heads, - mixtral_config.hidden_size / mixtral_config.num_attention_heads, - mixtral_config.hidden_size / mixtral_config.num_attention_heads, - 0.0f, /*dropout*/ - false, /*add_zero_attn*/ - DT_NONE, /*data_type*/ - nullptr, /*kernel_initializer*/ - mixtral_config.rotary_embedding_meta, - false, /*scaling query*/ - 1.0f, /*scaling factor*/ - true, /*qk_prod_scaling*/ - false, /*position_bias*/ - std::string("layers_" + std::to_string(i) + ".self_attn") - .c_str() /*name*/ - ); - break; - } case INC_DECODING_MODE: { mha = ff.inc_multiquery_self_attention( qkv_proj, @@ -196,7 +151,6 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::string("layers_" + std::to_string(i) + ".self_attn.o_proj") .c_str()); - // step 2: SILU activaion Tensor token_ff_norm[2] = {nullptr, nullptr}; ff.residual_rms_norm( @@ -211,10 +165,9 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, .c_str()); token = token_ff_norm[0]; Tensor ff_norm = token_ff_norm[1]; - // MoE - - Tensor w1 = ff.dense(ff_norm, + Tensor w1 = ff.dense( + ff_norm, mixtral_config.intermediate_size, AC_MODE_NONE, false, @@ -224,11 +177,10 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers_" + std::to_string(i) + - "_block_sparse_moe_experts_0_w1") - .c_str()); + std::string("layers_" + std::to_string(i) + "_block_sparse_moe_experts_0_w1").c_str()); - Tensor w3 = ff.dense(ff_norm, + Tensor w3 = ff.dense( + ff_norm, mixtral_config.intermediate_size, AC_MODE_NONE, false, @@ -238,20 +190,12 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers_" + std::to_string(i) + - "_block_sparse_moe_experts_0_w3") - .c_str()); + std::string("layers_" + std::to_string(i) + "_block_sparse_moe_experts_0_w3").c_str()); - Tensor multi = - ff.sigmoid_silu_multi(w1, - w3); -// DT_NONE, -// std::string("layers_" + std::to_string(i) + -// "_block_sparse_moe_experts_" + -// std::to_string(expert_idx) + "ssm") -// .c_str()); + Tensor multi = ff.sigmoid_silu_multi(w1, w3); //DT_NONE,std::string("layers_" + std::to_string(i) +"_block_sparse_moe_experts_" +std::to_string(expert_idx) + "ssm").c_str()); - mlp_out = ff.dense(multi, + mlp_out = ff.dense( + multi, mixtral_config.hidden_size, AC_MODE_NONE, false, @@ -261,10 +205,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers_" + std::to_string(i) + - "_block_sparse_moe_experts_0_w2") - .c_str()); - + std::string("layers_" + std::to_string(i) + "_block_sparse_moe_experts_0_w2").c_str()); } // final normalization and linear Tensor final_rms_norm_output[2] = {nullptr, nullptr}; @@ -290,23 +231,15 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, "lm_head"); Tensor output; - if (mode == BEAM_SEARCH_MODE) { - Tensor softmax = ff.softmax(dense, -1); - // output = ff.beam_top_k(softmax, mixtral_config.max_beam_width, false); - // output = ff.argmax(softmax, /*beam_Search*/ true); - output = ff.arg_top_k(softmax, mixtral_config.max_beam_width, false, true); - // output = ff.top_k(softmax, ) - } else { // Tensor softmax = ff.softmax(dense, -1); if (generation_config.do_sample) { dense = ff.scalar_truediv(dense, generation_config.temperature, false); Tensor softmax = ff.softmax(dense, -1); output = ff.sampling(softmax, generation_config.topp); } else { - // output = ff.arg_top_k(dense, /*k=*/1, false); - output = ff.argmax(dense, /*beam_Search*/ false); + Tensor softmax = ff.softmax(dense, -1); // TODO added that to copy llama + output = ff.argmax(softmax, /*beam_Search*/ false); } - } FileDataLoader *fileloader = new FileDataLoader( "", From a65e733b25a3dcaffe7b2d0379a2a55641ee827d Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 12:21:58 -0500 Subject: [PATCH 030/106] register tokenizer for mixtral --- src/runtime/request_manager.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/runtime/request_manager.cc b/src/runtime/request_manager.cc index fddaae09ce..668f76949b 100644 --- a/src/runtime/request_manager.cc +++ b/src/runtime/request_manager.cc @@ -26,6 +26,7 @@ #include #include #include +#include // For assert namespace FlexFlow { @@ -186,7 +187,7 @@ void RequestManager::register_tokenizer(ModelType type, this->eos_token_ids = eos_token_ids; std::filesystem::path tokenizer_folder(path); - if (model_type == ModelType::LLAMA) { + if (model_type == ModelType::LLAMA || model_type == ModelType::MIXTRAL) { // try with tokenizer.json first std::filesystem::path tokenizer_json_path; if (std::filesystem::is_directory(tokenizer_folder)) { @@ -366,7 +367,9 @@ RequestManager::RequestGuid request_.benchmarking_tokens, 15); // insert random number } else { - std::vector tokens = this->tokenizer_->Encode(request_.prompt); + std::vector tokens; + assert(this->tokenizer_ != nullptr && "Tokenizer is null!"); + tokens = this->tokenizer_->Encode(request_.prompt); // from here on, we will only use the max_length parameter if (request.max_new_tokens != -1) { request.max_length = tokens.size() + request.max_new_tokens; From 62bf0121f0a6d6fa1799aa71a57567414b1e2846 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 12:44:17 -0500 Subject: [PATCH 031/106] sync --- inference/models/mixtral.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index a082e6b7c8..79ba02af7c 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -75,7 +75,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.rms_norm_eps, mixtral_config.hidden_size, DT_NONE, - std::string("layers_" + std::to_string(i) + ".input_layernorm") + std::string("layers_" + std::to_string(i) + "_input_layernorm") .c_str()); } else { ff.residual_rms_norm( @@ -86,7 +86,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.hidden_size, false, // inplace_residual DT_NONE, - std::string("layers_" + std::to_string(i) + ".input_layernorm") + std::string("layers_" + std::to_string(i) + "_input_layernorm") .c_str()); token = token_att_norm[0]; att_norm = token_att_norm[1]; @@ -104,7 +104,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, // ? REG_MODE_NONE, // no regularization 0.0f, // no dropout - std::string("layers_" + std::to_string(i) + ".self_attn.qkv_proj") + std::string("layers_" + std::to_string(i) + "_self_attn_qkv_proj") .c_str()); Tensor mha; @@ -126,7 +126,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 1.0f, /*scaling factor*/ true, /*qk_prod_scaling*/ false, /*position_bias*/ - std::string("layers_" + std::to_string(i) + ".self_attn") + std::string("layers_" + std::to_string(i) + "_self_attn") .c_str() /*name*/ ); break; @@ -148,7 +148,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers_" + std::to_string(i) + ".self_attn.o_proj") + std::string("layers_" + std::to_string(i) + "_self_attn_o_proj") .c_str()); // step 2: SILU activaion @@ -161,7 +161,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.hidden_size, false, // inplace_residual DT_NONE, - std::string("layers_" + std::to_string(i) + ".post_attention_layernorm") + std::string("layers_" + std::to_string(i) + "_post_attention_layernorm") .c_str()); token = token_ff_norm[0]; Tensor ff_norm = token_ff_norm[1]; From 61adc0fc4488f5a73e61abed4ab5589e217d0d76 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 13:46:19 -0500 Subject: [PATCH 032/106] rename weights --- rename.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 rename.sh diff --git a/rename.sh b/rename.sh new file mode 100644 index 0000000000..daafadcfb3 --- /dev/null +++ b/rename.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Target directory +TARGET_DIR="/root/.cache/flexflow/weights/m4-ai/tinymistral-6x248m/half-precision" + +# Loop through files containing "layer_" +for file in "$TARGET_DIR"/*layer_*; do + if [[ -f "$file" ]]; then + mv "$file" "${file//layer_/layer.}" + fi +done + +# Loop through files containing "_input" +for file in "$TARGET_DIR"/*_input*; do + if [[ -f "$file" ]]; then + mv "$file" "${file//_input/_input}" + fi +done + +# Loop through files containing ".mlp." +for file in "$TARGET_DIR"/*.mlp.*; do + if [[ -f "$file" ]]; then + mv "$file" "${file//.mlp./_mlp_}" + fi +done From 8c69b8bc742f90531821a3ba69908dd3e983d486 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 13:50:22 -0500 Subject: [PATCH 033/106] sync --- inference/models/mixtral.cc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 79ba02af7c..37ed8587df 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -75,7 +75,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.rms_norm_eps, mixtral_config.hidden_size, DT_NONE, - std::string("layers_" + std::to_string(i) + "_input_layernorm") + std::string("layer." + std::to_string(i) + ".input_layernorm") .c_str()); } else { ff.residual_rms_norm( @@ -86,7 +86,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.hidden_size, false, // inplace_residual DT_NONE, - std::string("layers_" + std::to_string(i) + "_input_layernorm") + std::string("layer." + std::to_string(i) + ".input_layernorm") .c_str()); token = token_att_norm[0]; att_norm = token_att_norm[1]; @@ -104,7 +104,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, // ? REG_MODE_NONE, // no regularization 0.0f, // no dropout - std::string("layers_" + std::to_string(i) + "_self_attn_qkv_proj") + std::string("layer." + std::to_string(i) + ".self_attn_qkv_proj") .c_str()); Tensor mha; @@ -126,7 +126,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 1.0f, /*scaling factor*/ true, /*qk_prod_scaling*/ false, /*position_bias*/ - std::string("layers_" + std::to_string(i) + "_self_attn") + std::string("layer." + std::to_string(i) + ".self_attn") .c_str() /*name*/ ); break; @@ -148,7 +148,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers_" + std::to_string(i) + "_self_attn_o_proj") + std::string("layer." + std::to_string(i) + ".self_attn_o_proj") .c_str()); // step 2: SILU activaion @@ -161,7 +161,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.hidden_size, false, // inplace_residual DT_NONE, - std::string("layers_" + std::to_string(i) + "_post_attention_layernorm") + std::string("layer." + std::to_string(i) + ".post_attention_layernorm") .c_str()); token = token_ff_norm[0]; Tensor ff_norm = token_ff_norm[1]; @@ -177,7 +177,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers_" + std::to_string(i) + "_block_sparse_moe_experts_0_w1").c_str()); + std::string("layer." + std::to_string(i) + ".block_sparse_moe_experts_0_w1").c_str()); Tensor w3 = ff.dense( ff_norm, @@ -190,9 +190,9 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers_" + std::to_string(i) + "_block_sparse_moe_experts_0_w3").c_str()); + std::string("layer." + std::to_string(i) + ".block_sparse_moe_experts_0_w3").c_str()); - Tensor multi = ff.sigmoid_silu_multi(w1, w3); //DT_NONE,std::string("layers_" + std::to_string(i) +"_block_sparse_moe_experts_" +std::to_string(expert_idx) + "ssm").c_str()); + Tensor multi = ff.sigmoid_silu_multi(w1, w3); //DT_NONE,std::string("layer." + std::to_string(i) +".block_sparse_moe_experts." +std::to_string(expert_idx) + "ssm").c_str()); mlp_out = ff.dense( multi, @@ -205,7 +205,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers_" + std::to_string(i) + "_block_sparse_moe_experts_0_w2").c_str()); + std::string("layer." + std::to_string(i) + ".block_sparse_moe_experts_0_w2").c_str()); } // final normalization and linear Tensor final_rms_norm_output[2] = {nullptr, nullptr}; From 0b8916910de518cc3a7c9cc46eb29a270133e39e Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 14:01:47 -0500 Subject: [PATCH 034/106] sync --- rename.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/rename.sh b/rename.sh index daafadcfb3..9c0e1173e9 100644 --- a/rename.sh +++ b/rename.sh @@ -13,13 +13,26 @@ done # Loop through files containing "_input" for file in "$TARGET_DIR"/*_input*; do if [[ -f "$file" ]]; then - mv "$file" "${file//_input/_input}" + mv "$file" "${file//_input/.input}" fi done # Loop through files containing ".mlp." -for file in "$TARGET_DIR"/*.mlp.*; do +for file in "$TARGET_DIR"/*_mlp_*; do if [[ -f "$file" ]]; then - mv "$file" "${file//.mlp./_mlp_}" + mv "$file" "${file//.mlp./.mlp.}" + fi +done + +for file in "$TARGET_DIR"/*_post*; do + if [[ -f "$file" ]]; then + mv "$file" "${file//_post/.post}" + fi +done + + +for file in "$TARGET_DIR"/*_block*; do + if [[ -f "$file" ]]; then + mv "$file" "${file//_block/.block}" fi done From 76cac3631f793b28fb0e0472a585665d61c622a4 Mon Sep 17 00:00:00 2001 From: hugo Date: Sat, 7 Dec 2024 19:02:19 +0000 Subject: [PATCH 035/106] permission --- rename.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 rename.sh diff --git a/rename.sh b/rename.sh old mode 100644 new mode 100755 From 53a4cc4ec8936a254c10f43f60858b06fa2082bd Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 14:05:16 -0500 Subject: [PATCH 036/106] sync --- rename.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rename.sh b/rename.sh index 9c0e1173e9..3cf895b784 100644 --- a/rename.sh +++ b/rename.sh @@ -36,3 +36,9 @@ for file in "$TARGET_DIR"/*_block*; do mv "$file" "${file//_block/.block}" fi done + +for file in "$TARGET_DIR"/*layers_*; do + if [[ -f "$file" ]]; then + mv "$file" "${file//layers_/layers.}" + fi +done \ No newline at end of file From aeb29e9f66f602b31cb11192e4e9b36d19ca2ba1 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 14:07:10 -0500 Subject: [PATCH 037/106] sync --- inference/models/mixtral.cc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 37ed8587df..7bf8702b8f 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -75,7 +75,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.rms_norm_eps, mixtral_config.hidden_size, DT_NONE, - std::string("layer." + std::to_string(i) + ".input_layernorm") + std::string("layers." + std::to_string(i) + ".input_layernorm") .c_str()); } else { ff.residual_rms_norm( @@ -86,7 +86,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.hidden_size, false, // inplace_residual DT_NONE, - std::string("layer." + std::to_string(i) + ".input_layernorm") + std::string("layers." + std::to_string(i) + ".input_layernorm") .c_str()); token = token_att_norm[0]; att_norm = token_att_norm[1]; @@ -104,7 +104,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, // ? REG_MODE_NONE, // no regularization 0.0f, // no dropout - std::string("layer." + std::to_string(i) + ".self_attn_qkv_proj") + std::string("layers." + std::to_string(i) + ".self_attn_qkv_proj") .c_str()); Tensor mha; @@ -126,7 +126,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 1.0f, /*scaling factor*/ true, /*qk_prod_scaling*/ false, /*position_bias*/ - std::string("layer." + std::to_string(i) + ".self_attn") + std::string("layers." + std::to_string(i) + ".self_attn") .c_str() /*name*/ ); break; @@ -148,7 +148,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layer." + std::to_string(i) + ".self_attn_o_proj") + std::string("layers." + std::to_string(i) + ".self_attn_o_proj") .c_str()); // step 2: SILU activaion @@ -161,7 +161,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mixtral_config.hidden_size, false, // inplace_residual DT_NONE, - std::string("layer." + std::to_string(i) + ".post_attention_layernorm") + std::string("layers." + std::to_string(i) + ".post_attention_layernorm") .c_str()); token = token_ff_norm[0]; Tensor ff_norm = token_ff_norm[1]; @@ -177,7 +177,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layer." + std::to_string(i) + ".block_sparse_moe_experts_0_w1").c_str()); + std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_0_w1").c_str()); Tensor w3 = ff.dense( ff_norm, @@ -190,9 +190,9 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layer." + std::to_string(i) + ".block_sparse_moe_experts_0_w3").c_str()); + std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_0_w3").c_str()); - Tensor multi = ff.sigmoid_silu_multi(w1, w3); //DT_NONE,std::string("layer." + std::to_string(i) +".block_sparse_moe_experts." +std::to_string(expert_idx) + "ssm").c_str()); + Tensor multi = ff.sigmoid_silu_multi(w1, w3); //DT_NONE,std::string("layers." + std::to_string(i) +".block_sparse_moe_experts." +std::to_string(expert_idx) + "ssm").c_str()); mlp_out = ff.dense( multi, @@ -205,7 +205,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layer." + std::to_string(i) + ".block_sparse_moe_experts_0_w2").c_str()); + std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_0_w2").c_str()); } // final normalization and linear Tensor final_rms_norm_output[2] = {nullptr, nullptr}; From d27804fa11969b4ec7ed9aea5c5f62b74460c0c6 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 14:08:49 -0500 Subject: [PATCH 038/106] sync --- rename.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rename.sh b/rename.sh index 3cf895b784..9c3c583987 100755 --- a/rename.sh +++ b/rename.sh @@ -41,4 +41,10 @@ for file in "$TARGET_DIR"/*layers_*; do if [[ -f "$file" ]]; then mv "$file" "${file//layers_/layers.}" fi +done + +for file in "$TARGET_DIR"/*proj_*; do + if [[ -f "$file" ]]; then + mv "$file" "${file//proj_/proj.}" + fi done \ No newline at end of file From ecb9675184bd884c858855e1e26d23f361c831ec Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 14:11:14 -0500 Subject: [PATCH 039/106] sync --- rename.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rename.sh b/rename.sh index 9c3c583987..63e7d5f5a3 100755 --- a/rename.sh +++ b/rename.sh @@ -47,4 +47,16 @@ for file in "$TARGET_DIR"/*proj_*; do if [[ -f "$file" ]]; then mv "$file" "${file//proj_/proj.}" fi +done + +for file in "$TARGET_DIR"/*_self*; do + if [[ -f "$file" ]]; then + mv "$file" "${file//_self/.self}" + fi +done + +for file in "$TARGET_DIR"/*attn_*; do + if [[ -f "$file" ]]; then + mv "$file" "${file//attn_/attn.}" + fi done \ No newline at end of file From af665bd2a832d09de520f762623a8c13efcdc237 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 14:21:35 -0500 Subject: [PATCH 040/106] which loading --- src/runtime/file_loader.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/runtime/file_loader.cc b/src/runtime/file_loader.cc index 6ffa9370f0..a2cd63a4ee 100644 --- a/src/runtime/file_loader.cc +++ b/src/runtime/file_loader.cc @@ -96,7 +96,7 @@ void load_attention_o_proj_bias_to_dense_v2(DT *ptr, // assert(num_heads == num_kv_heads); int idx = 0; - std::cout << "Loading weight file " << filename << std::endl; + std::cout << "Loading weight file " << filename << " in load_attention_o_proj_bias_to_dense_v2 " << std::endl; std::string weight_filepath = join_path({weights_folder, filename}); int n_heads = num_heads; @@ -168,7 +168,7 @@ void load_attention_bias_v2(DT *ptr, int file_index = 0; for (auto filename : bias_files) { - std::cout << "Loading weight file " << filename << std::endl; + std::cout << "Loading weight file " << filename << " in load_attention_bias_v2 " << std::endl; std::string weight_filepath = join_path({weights_folder, filename}); int n_heads = file_index == 0 ? num_heads : num_kv_heads; @@ -254,7 +254,7 @@ void load_attention_weights_to_dense_v2(DT *ptr, tensor_parallelism_degree; if (!load_o_proj) { for (auto filename : weight_filenames) { - std::cout << "Loading weight file " << filename << std::endl; + std::cout << "Loading weight file " << filename << " in load_attention_weights_to_dense_v2 (no load_o_proj) " << std::endl; std::string weight_filepath = join_path({weights_folder, filename}); int data_index = 0; @@ -308,7 +308,7 @@ void load_attention_weights_to_dense_v2(DT *ptr, assert(base_index == (q_size + k_replicate_size + v_replicate_size) / tensor_parallelism_degree); } else { - std::cout << "Loading weight file " << o_file << std::endl; + std::cout << "Loading weight file " << o_file << " in load_attention_weights_to_dense_v2 (no load_o_proj) " << std::endl; std::string weight_filepath = join_path({weights_folder, o_file}); std::ifstream in(weight_filepath, std::ios::in | std::ios::binary); @@ -443,7 +443,7 @@ void load_attention_weights_quantized(char *ptr, // q, k, v, o -> 0, 1, 2, 3 for (auto filename : weight_filenames) { - std::cout << "Loading weight file " << filename << std::endl; + std::cout << "Loading weight file " << filename << " in load_attention_weights_quantized (qkvo)" << std::endl; std::string weight_filepath = join_path({weights_folder, filename}); size_t partial_size = one_weight_file_size; @@ -493,7 +493,7 @@ void load_attention_weights_quantized(char *ptr, size_t offset = data_type == DT_INT8 ? one_weight_file_size * 4 : (one_weight_file_size * 4) / 2; for (auto filename : weight_filenames) { - std::cout << "Loading weight file " << filename << std::endl; + std::cout << "Loading weight file " << filename << " in load_attention_weights_quantized (scale and offset) " << std::endl; std::string weight_filepath = join_path({weights_folder, filename}); for (int i = 0; i < 2; i++) { @@ -838,7 +838,7 @@ void FileDataLoader::load_single_weight_tensor(FFModel *ff, weight_filename += (weight_idx == 0) ? ".attn_bias" : ((weight_idx == 1) ? ".weight" : ".bias"); - std::cout << "Loading weight file " << weight_filename << std::endl; + std::cout << "Loading weight file " << weight_filename << "in load_single_weight_tensor, OP_ADD_BIAS_RESIDUAL_LAYERNORM " << std::endl; std::string weight_filepath = join_path({weights_folder, weight_filename}); load_from_file(data, volume, weight_filepath); @@ -849,7 +849,7 @@ void FileDataLoader::load_single_weight_tensor(FFModel *ff, if (weight_filename != "embed_tokens_weight_lm_head") { weight_filename += weight_idx == 0 ? ".weight" : ".bias"; } - std::cout << "Loading weight file " << weight_filename << std::endl; + std::cout << "Loading weight file " << weight_filename << "in load_single_weight_tensor, default op" << std::endl; std::string weight_filepath = join_path({weights_folder, weight_filename}); load_from_file(data, volume, weight_filepath); From 16ab9126b1194bdef222aceaef0ea10e40cf03e2 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 14:25:46 -0500 Subject: [PATCH 041/106] sync --- src/runtime/file_loader.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/file_loader.cc b/src/runtime/file_loader.cc index a2cd63a4ee..28992fc8e8 100644 --- a/src/runtime/file_loader.cc +++ b/src/runtime/file_loader.cc @@ -451,7 +451,7 @@ void load_attention_weights_quantized(char *ptr, if (!in.good()) { std::cout << "Could not open file: " << weight_filepath << std::endl; } - assert(in.good() && "incorrect weight file path"); + assert(in.good() && "incorrect weight file path (QKVO)"); std::vector host_array(partial_size); size_t loaded_data_size = sizeof(char) * partial_size; in.seekg(0, in.end); @@ -505,7 +505,7 @@ void load_attention_weights_quantized(char *ptr, if (!in.good()) { std::cout << "Could not open file: " << meta_file << std::endl; } - assert(in.good() && "incorrect weight file path"); + assert(in.good() && "incorrect weight file path (scale and offset)"); if (use_full_precision) { // float From c3945e38379512c1d4f1efbe4b479f799516d145 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 14:49:31 -0500 Subject: [PATCH 042/106] sync --- inference/models/mixtral.cc | 2 +- src/runtime/file_loader.cc | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 7bf8702b8f..af45522e87 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -104,7 +104,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, // ? REG_MODE_NONE, // no regularization 0.0f, // no dropout - std::string("layers." + std::to_string(i) + ".self_attn_qkv_proj") + std::string("layers." + std::to_string(i) + ".self_attn.qkv_proj") .c_str()); Tensor mha; diff --git a/src/runtime/file_loader.cc b/src/runtime/file_loader.cc index 28992fc8e8..f89d26618b 100644 --- a/src/runtime/file_loader.cc +++ b/src/runtime/file_loader.cc @@ -586,7 +586,7 @@ void load_from_quantized_file(char *ptr, if (!in.good()) { std::cout << "Could not open file: " << file << std::endl; } - assert(in.good() && "incorrect weight file path"); + assert(in.good() && "incorrect weight file path (quantized file)"); // value file, every element is in one byte if (file_idx == 0) { @@ -754,6 +754,7 @@ void FileDataLoader::load_single_weight_tensor(FFModel *ff, // assert(data_type_size(weight->data_type) == sizeof(DT)); DT *data = (DT *)malloc(sizeof(DT) * volume); + // llama has l-name = "layers.0.self_attn.qkv_proj_1000003" while tinymistral has l-name = "layers.0.self_attn_qkv_proj" std::string weight_filename = removeGuidOperatorName(std::string(l->name)); bool is_attn_proj = false, is_o_proj = false; From 9385e8239e015456e0fa476b95ecb335373a56ce Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 14:55:19 -0500 Subject: [PATCH 043/106] .o --- inference/models/mixtral.cc | 2 +- src/runtime/file_loader.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index af45522e87..20e96b5a6d 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -148,7 +148,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers." + std::to_string(i) + ".self_attn_o_proj") + std::string("layers." + std::to_string(i) + ".self_attn.o_proj") .c_str()); // step 2: SILU activaion diff --git a/src/runtime/file_loader.cc b/src/runtime/file_loader.cc index f89d26618b..915efca167 100644 --- a/src/runtime/file_loader.cc +++ b/src/runtime/file_loader.cc @@ -839,7 +839,7 @@ void FileDataLoader::load_single_weight_tensor(FFModel *ff, weight_filename += (weight_idx == 0) ? ".attn_bias" : ((weight_idx == 1) ? ".weight" : ".bias"); - std::cout << "Loading weight file " << weight_filename << "in load_single_weight_tensor, OP_ADD_BIAS_RESIDUAL_LAYERNORM " << std::endl; + std::cout << "Loading weight file " << weight_filename << " in load_single_weight_tensor, OP_ADD_BIAS_RESIDUAL_LAYERNORM " << std::endl; std::string weight_filepath = join_path({weights_folder, weight_filename}); load_from_file(data, volume, weight_filepath); @@ -850,7 +850,7 @@ void FileDataLoader::load_single_weight_tensor(FFModel *ff, if (weight_filename != "embed_tokens_weight_lm_head") { weight_filename += weight_idx == 0 ? ".weight" : ".bias"; } - std::cout << "Loading weight file " << weight_filename << "in load_single_weight_tensor, default op" << std::endl; + std::cout << "Loading weight file " << weight_filename << " in load_single_weight_tensor, default op" << std::endl; std::string weight_filepath = join_path({weights_folder, weight_filename}); load_from_file(data, volume, weight_filepath); From ce919660c245ff6b20367fa7758355dce390d6fe Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 15:38:59 -0500 Subject: [PATCH 044/106] sync --- src/ops/inc_multihead_self_attention.cu | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ops/inc_multihead_self_attention.cu b/src/ops/inc_multihead_self_attention.cu index 454926bcdb..e3565179a5 100644 --- a/src/ops/inc_multihead_self_attention.cu +++ b/src/ops/inc_multihead_self_attention.cu @@ -908,7 +908,12 @@ void compute_attention_kernel_generation(IncMultiHeadSelfAttentionMeta const *m, int const per_head_size = m->qProjSize; float scale = (*m->qk_prod_scaling) ? 1.0f / sqrt(m->kProjSize) : 1.0f; size_t smem_sz; - if (per_head_size == 64) { + if (per_head_size == 32) { + constexpr int THREADS_PER_VALUE_32 = threads_per_value_t::value; + LAUNCH_ATTENTION_SCORE_KERNEL( + DT, 32, 32, 4, THREADS_PER_VALUE_32, 128, stream); + } + else if (per_head_size == 64) { constexpr int THREADS_PER_VALUE_64 = threads_per_value_t::value; LAUNCH_ATTENTION_SCORE_KERNEL( DT, 64, 64, 4, THREADS_PER_VALUE_64, 128, stream); From 7e558bc562360c5038670ef26faa36b4fb6c56ff Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 15:43:09 -0500 Subject: [PATCH 045/106] able to output with mixtral (!!!) but it's all etc etc etc --- inference/models/mixtral.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 20e96b5a6d..7a704cca3e 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -165,6 +165,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, .c_str()); token = token_ff_norm[0]; Tensor ff_norm = token_ff_norm[1]; + // MoE Tensor w1 = ff.dense( ff_norm, From c8007fd3051cf1d74a27f296622e05fb646c1445 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 15:44:46 -0500 Subject: [PATCH 046/106] try expert 1 --- inference/models/mixtral.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 7a704cca3e..f4485deda1 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -178,7 +178,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_0_w1").c_str()); + std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w1").c_str()); Tensor w3 = ff.dense( ff_norm, @@ -191,7 +191,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_0_w3").c_str()); + std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w3").c_str()); Tensor multi = ff.sigmoid_silu_multi(w1, w3); //DT_NONE,std::string("layers." + std::to_string(i) +".block_sparse_moe_experts." +std::to_string(expert_idx) + "ssm").c_str()); @@ -206,7 +206,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_0_w2").c_str()); + std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w2").c_str()); } // final normalization and linear Tensor final_rms_norm_output[2] = {nullptr, nullptr}; From aa01156669f1e8d65cb81e56a9510e3c7c38bded Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 16:24:16 -0500 Subject: [PATCH 047/106] revert experts --- inference/models/mixtral.cc | 147 +++++++++++++++++++++++++++--------- 1 file changed, 110 insertions(+), 37 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index f4485deda1..fc4b240d92 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -167,47 +167,120 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor ff_norm = token_ff_norm[1]; // MoE - Tensor w1 = ff.dense( - ff_norm, - mixtral_config.intermediate_size, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w1").c_str()); + Tensor gate = ff.dense( + ff_norm, + mixtral_config.num_local_experts, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") + .c_str()); + gate = ff.softmax( + gate, + 0, + DT_NONE, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_softmax") + .c_str()); - Tensor w3 = ff.dense( - ff_norm, - mixtral_config.intermediate_size, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w3").c_str()); + Tensor topk_out[2] = {nullptr, nullptr}; + ff.top_k( + gate, + topk_out, + mixtral_config.num_experts_per_tok, + false, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") + .c_str()); + Tensor topk_values = topk_out[0]; + Tensor topk_indices = topk_out[1]; - Tensor multi = ff.sigmoid_silu_multi(w1, w3); //DT_NONE,std::string("layers." + std::to_string(i) +".block_sparse_moe_experts." +std::to_string(expert_idx) + "ssm").c_str()); + Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; + ff.group_by( + ff_norm, + topk_indices, + grouped_tokens, + mixtral_config.num_local_experts, + 0.0f, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby") + .c_str()); - mlp_out = ff.dense( - multi, - mixtral_config.hidden_size, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w2").c_str()); + Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; + for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; + expert_idx++) { + Tensor w1 = ff.dense(grouped_tokens[expert_idx], + mixtral_config.intermediate_size, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers." + std::to_string(i) + + ".block_sparse_moe_experts_" + + std::to_string(expert_idx) + "_w1") + .c_str()); + + Tensor w3 = ff.dense(grouped_tokens[expert_idx], + mixtral_config.intermediate_size, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers." + std::to_string(i) + + ".block_sparse_moe_experts_" + + std::to_string(expert_idx) + "_w3") + .c_str()); + + Tensor multi = + ff.sigmoid_silu_multi(w1, + w3, + DT_NONE, + std::string("layers." + std::to_string(i) + + ".block_sparse_moe_experts_" + + std::to_string(expert_idx) + "ssm") + .c_str()); + + Tensor w2 = ff.dense(multi, + mixtral_config.hidden_size, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers." + std::to_string(i) + + ".block_sparse_moe_experts_" + + std::to_string(expert_idx) + "_w2") + .c_str()); + aggregate_inputs[4 + expert_idx] = w2; + } + + Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); + topk_values = ff.divide(topk_values, topk_values_reduced); + + aggregate_inputs[0] = topk_values; + aggregate_inputs[1] = topk_indices; + aggregate_inputs[2] = aggregate_inputs[3] = nullptr; + mlp_out = ff.aggregate(aggregate_inputs, + mixtral_config.num_local_experts, + 0.0f, + std::string("layers." + std::to_string(i) + + ".block_sparse_moe_experts_aggregate") + .c_str()); } + // final normalization and linear Tensor final_rms_norm_output[2] = {nullptr, nullptr}; ff.residual_rms_norm(token, From 539b49180951a76ec02f1507e0938a6262b4003b Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 16:28:01 -0500 Subject: [PATCH 048/106] tmp fix --- inference/models/mixtral.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index fc4b240d92..36872828e0 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -272,7 +272,9 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[0] = topk_values; aggregate_inputs[1] = topk_indices; - aggregate_inputs[2] = aggregate_inputs[3] = nullptr; + aggregate_inputs[2] = topk_values; // TODO this is a tmp fix + aggregate_inputs[3] = gate; // TODO this is a tmp fix +// aggregate_inputs[2] = aggregate_inputs[3] = nullptr; mlp_out = ff.aggregate(aggregate_inputs, mixtral_config.num_local_experts, 0.0f, From 28b2df0333666ae9c7bfccb3fa48e638569637c1 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 17:13:32 -0500 Subject: [PATCH 049/106] dummy gate --- inference/models/mixtral.cc | 25 ++++++++++++++++++++++--- src/ops/aggregate.cc | 2 +- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 36872828e0..4380da664a 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -209,8 +209,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, .c_str()); Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; - for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; - expert_idx++) { + for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; expert_idx++) { Tensor w1 = ff.dense(grouped_tokens[expert_idx], mixtral_config.intermediate_size, AC_MODE_NONE, @@ -270,10 +269,30 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); topk_values = ff.divide(topk_values, topk_values_reduced); + Tensor dummy_gate = ff.dense( + ff_norm, + mixtral_config.num_local_experts, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") + .c_str()); + + dummy_gate = ff.softmax( + gate, + 0, + DT_NONE, + std::string("dummy_gate").c_str()); + aggregate_inputs[0] = topk_values; aggregate_inputs[1] = topk_indices; aggregate_inputs[2] = topk_values; // TODO this is a tmp fix - aggregate_inputs[3] = gate; // TODO this is a tmp fix + aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix // aggregate_inputs[2] = aggregate_inputs[3] = nullptr; mlp_out = ff.aggregate(aggregate_inputs, mixtral_config.num_local_experts, diff --git a/src/ops/aggregate.cc b/src/ops/aggregate.cc index c83b738a0e..ffa988410c 100644 --- a/src/ops/aggregate.cc +++ b/src/ops/aggregate.cc @@ -47,7 +47,7 @@ Tensor FFModel::aggregate( OP_AGGREGATE, DT_FLOAT, name, - n + 4 /*inputs*/, + n + 4 /*num inputs*/, 0 /*weights*/, 1 /*outputs*/, inputs); From 8ad9478831c194aa2fe7195c9c52ea8068bb333a Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 17:19:35 -0500 Subject: [PATCH 050/106] bad softmax fix --- src/ops/softmax.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ops/softmax.cc b/src/ops/softmax.cc index a02d88b98b..ff8ee6cc5b 100644 --- a/src/ops/softmax.cc +++ b/src/ops/softmax.cc @@ -148,7 +148,7 @@ Softmax::Softmax(FFModel &model, _input), dim(_dim) { // Currently assume we always perform softmax along the inner most dim - assert(dim == 0); +// assert(dim == 0); // TODO actually allow this!! layer_guid = _layer_guid; ParallelDim dims[MAX_TENSOR_DIM]; int numdim = _input->num_dims; From 9dcb5c2fd072c3a8eacb116d1ff0f53e1600d7c7 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 20:19:45 -0500 Subject: [PATCH 051/106] printf --- inference/models/mixtral.cc | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 4380da664a..b5613c29d4 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -63,6 +63,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor mlp_out = nullptr; for (int i = 0; i < mixtral_config.num_hidden_layers; i++) { + printf("mixtral hidden layer %d\n", i); + // set transformer layer id ff.set_transformer_layer_id(i); @@ -167,6 +169,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor ff_norm = token_ff_norm[1]; // MoE + printf("moe's input, ff_norm, has number of dims %d\n", ff_norm->num_dims); +// printf("moe's input, ff_norm, has shape: %d, %d\n", ff_norm->dims[0], ff_norm->dims[1]); Tensor gate = ff.dense( ff_norm, mixtral_config.num_local_experts, @@ -180,23 +184,25 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 0.0f, std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") .c_str()); - gate = ff.softmax( + + printf("gate has number of dims %d\n", gate->num_dims); + gate = ff.softmax( // This operation fails! gate, 0, DT_NONE, std::string("layers." + std::to_string(i) + ".block_sparse_moe_softmax") .c_str()); - Tensor topk_out[2] = {nullptr, nullptr}; + Tensor topk_out[2] = {nullptr, nullptr}; // (2,) ff.top_k( gate, - topk_out, + topk_out, // (2,) mixtral_config.num_experts_per_tok, false, std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") .c_str()); - Tensor topk_values = topk_out[0]; - Tensor topk_indices = topk_out[1]; + Tensor topk_values = topk_out[0]; printf("topk_values has number of dims %d\n", topk_values->num_dims); + Tensor topk_indices = topk_out[1]; printf("topk_indices has number of dims %d\n", topk_indices->num_dims); Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; ff.group_by( @@ -207,7 +213,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 0.0f, std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby") .c_str()); - + printf("grouped_tokens[0] has number of dims %d\n", grouped_tokens[0]->num_dims); Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; expert_idx++) { Tensor w1 = ff.dense(grouped_tokens[expert_idx], @@ -263,11 +269,12 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, ".block_sparse_moe_experts_" + std::to_string(expert_idx) + "_w2") .c_str()); + printf("w2 and aggreagte_inputs[%d] has number of dims %d\n",4 + expert_idx, w2->num_dims); aggregate_inputs[4 + expert_idx] = w2; } - Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); - topk_values = ff.divide(topk_values, topk_values_reduced); + Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); printf("topk_values_reduced has number of dims %d\n", topk_values_reduced->num_dims); + topk_values = ff.divide(topk_values, topk_values_reduced); printf("topk_values has number of dims %d\n", topk_values->num_dims); Tensor dummy_gate = ff.dense( ff_norm, @@ -289,8 +296,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, DT_NONE, std::string("dummy_gate").c_str()); - aggregate_inputs[0] = topk_values; - aggregate_inputs[1] = topk_indices; + aggregate_inputs[0] = topk_values; printf("aggregate_inputs[0] has number of dims %d\n", aggregate_inputs[0]->num_dims); + aggregate_inputs[1] = topk_indices; printf("aggregate_inputs[1] has number of dims %d\n", aggregate_inputs[1]->num_dims); aggregate_inputs[2] = topk_values; // TODO this is a tmp fix aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix // aggregate_inputs[2] = aggregate_inputs[3] = nullptr; From 7ed7d6501be75dd52fbff068011f5a3b05815050 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 20:25:53 -0500 Subject: [PATCH 052/106] dims --- inference/models/mixtral.cc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index b5613c29d4..f673af559b 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -169,7 +169,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor ff_norm = token_ff_norm[1]; // MoE - printf("moe's input, ff_norm, has number of dims %d\n", ff_norm->num_dims); + printf("moe's input, ff_norm, has dims %d %d %d\n", ff_norm->dims[0], ff_norm->dims[1], ff_norm->dims[2]); // printf("moe's input, ff_norm, has shape: %d, %d\n", ff_norm->dims[0], ff_norm->dims[1]); Tensor gate = ff.dense( ff_norm, @@ -185,7 +185,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") .c_str()); - printf("gate has number of dims %d\n", gate->num_dims); + printf("gate has dims %d\n", gate->dims[0], gate->dims[1], gate->dims[2]); gate = ff.softmax( // This operation fails! gate, 0, @@ -201,8 +201,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, false, std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") .c_str()); - Tensor topk_values = topk_out[0]; printf("topk_values has number of dims %d\n", topk_values->num_dims); - Tensor topk_indices = topk_out[1]; printf("topk_indices has number of dims %d\n", topk_indices->num_dims); + Tensor topk_values = topk_out[0]; printf("topk_values has dims %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2]); + Tensor topk_indices = topk_out[1]; printf("topk_indices has dims %d\n", topk_indices->dims[0], topk_indices->dims[1], topk_indices->dims[2]); Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; ff.group_by( @@ -213,7 +213,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 0.0f, std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby") .c_str()); - printf("grouped_tokens[0] has number of dims %d\n", grouped_tokens[0]->num_dims); + printf("grouped_tokens[0] has dims %d\n", grouped_tokens[0]->dims[0], grouped_tokens[0]->dims[1], grouped_tokens[0]->dims[2]); Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; expert_idx++) { Tensor w1 = ff.dense(grouped_tokens[expert_idx], @@ -269,12 +269,12 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, ".block_sparse_moe_experts_" + std::to_string(expert_idx) + "_w2") .c_str()); - printf("w2 and aggreagte_inputs[%d] has number of dims %d\n",4 + expert_idx, w2->num_dims); + printf("w2 and aggreagte_inputs[%d] has dims %d\n",4 + expert_idx, w2->dims[0], w2->dims[1], w2->dims[2]); aggregate_inputs[4 + expert_idx] = w2; } - Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); printf("topk_values_reduced has number of dims %d\n", topk_values_reduced->num_dims); - topk_values = ff.divide(topk_values, topk_values_reduced); printf("topk_values has number of dims %d\n", topk_values->num_dims); + Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); printf("topk_values_reduced has dims %d\n", topk_values_reduced->dims[0], topk_values_reduced->dims[1], topk_values_reduced->dims[2]); + topk_values = ff.divide(topk_values, topk_values_reduced); printf("topk_values has dims %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2]); Tensor dummy_gate = ff.dense( ff_norm, @@ -296,8 +296,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, DT_NONE, std::string("dummy_gate").c_str()); - aggregate_inputs[0] = topk_values; printf("aggregate_inputs[0] has number of dims %d\n", aggregate_inputs[0]->num_dims); - aggregate_inputs[1] = topk_indices; printf("aggregate_inputs[1] has number of dims %d\n", aggregate_inputs[1]->num_dims); + aggregate_inputs[0] = topk_values; printf("aggregate_inputs[0] has dims %d\n", aggregate_inputs[0]->dims[0], aggregate_inputs[0]->dims[1], aggregate_inputs[0]->dims[2]); + aggregate_inputs[1] = topk_indices; printf("aggregate_inputs[1] has dims %d\n", aggregate_inputs[1]->dims[0], aggregate_inputs[1]->dims[1], aggregate_inputs[1]->dims[2]); aggregate_inputs[2] = topk_values; // TODO this is a tmp fix aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix // aggregate_inputs[2] = aggregate_inputs[3] = nullptr; From a906c6ad12af9256522bf1b6e3d0e6a0ce08587a Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 20:28:45 -0500 Subject: [PATCH 053/106] sync --- inference/models/mixtral.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index f673af559b..3ee1073d1c 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -185,7 +185,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") .c_str()); - printf("gate has dims %d\n", gate->dims[0], gate->dims[1], gate->dims[2]); + printf("gate has dims %d %d %d\n", gate->dims[0], gate->dims[1], gate->dims[2]); gate = ff.softmax( // This operation fails! gate, 0, @@ -213,7 +213,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 0.0f, std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby") .c_str()); - printf("grouped_tokens[0] has dims %d\n", grouped_tokens[0]->dims[0], grouped_tokens[0]->dims[1], grouped_tokens[0]->dims[2]); + printf("grouped_tokens[0] has dims %d %d %d\n", grouped_tokens[0]->dims[0], grouped_tokens[0]->dims[1], grouped_tokens[0]->dims[2]); Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; expert_idx++) { Tensor w1 = ff.dense(grouped_tokens[expert_idx], @@ -269,12 +269,12 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, ".block_sparse_moe_experts_" + std::to_string(expert_idx) + "_w2") .c_str()); - printf("w2 and aggreagte_inputs[%d] has dims %d\n",4 + expert_idx, w2->dims[0], w2->dims[1], w2->dims[2]); + printf("w2 and aggreagte_inputs[%d] has dims %d %d %d\n",4 + expert_idx, w2->dims[0], w2->dims[1], w2->dims[2]); aggregate_inputs[4 + expert_idx] = w2; } - Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); printf("topk_values_reduced has dims %d\n", topk_values_reduced->dims[0], topk_values_reduced->dims[1], topk_values_reduced->dims[2]); - topk_values = ff.divide(topk_values, topk_values_reduced); printf("topk_values has dims %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2]); + Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); printf("topk_values_reduced has dims %d %d %d\n", topk_values_reduced->dims[0], topk_values_reduced->dims[1], topk_values_reduced->dims[2]); + topk_values = ff.divide(topk_values, topk_values_reduced); printf("topk_values has dims %d %d %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2]); Tensor dummy_gate = ff.dense( ff_norm, @@ -296,8 +296,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, DT_NONE, std::string("dummy_gate").c_str()); - aggregate_inputs[0] = topk_values; printf("aggregate_inputs[0] has dims %d\n", aggregate_inputs[0]->dims[0], aggregate_inputs[0]->dims[1], aggregate_inputs[0]->dims[2]); - aggregate_inputs[1] = topk_indices; printf("aggregate_inputs[1] has dims %d\n", aggregate_inputs[1]->dims[0], aggregate_inputs[1]->dims[1], aggregate_inputs[1]->dims[2]); + aggregate_inputs[0] = topk_values; printf("aggregate_inputs[0] has dims %d %d %d\n", aggregate_inputs[0]->dims[0], aggregate_inputs[0]->dims[1], aggregate_inputs[0]->dims[2]); + aggregate_inputs[1] = topk_indices; printf("aggregate_inputs[1] has dims %d %d %d\n", aggregate_inputs[1]->dims[0], aggregate_inputs[1]->dims[1], aggregate_inputs[1]->dims[2]); aggregate_inputs[2] = topk_values; // TODO this is a tmp fix aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix // aggregate_inputs[2] = aggregate_inputs[3] = nullptr; From 0af8064be20de6e1027d2ecb8136d56a0c3eb829 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 20:30:08 -0500 Subject: [PATCH 054/106] sync --- inference/models/mixtral.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 3ee1073d1c..8634550faa 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -201,8 +201,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, false, std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") .c_str()); - Tensor topk_values = topk_out[0]; printf("topk_values has dims %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2]); - Tensor topk_indices = topk_out[1]; printf("topk_indices has dims %d\n", topk_indices->dims[0], topk_indices->dims[1], topk_indices->dims[2]); + Tensor topk_values = topk_out[0]; printf("topk_values has dims %d %d %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2]); + Tensor topk_indices = topk_out[1]; printf("topk_indices has dims %d %d %d\n", topk_indices->dims[0], topk_indices->dims[1], topk_indices->dims[2]); Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; ff.group_by( From 4d26fb5b35b6bf56fdf81a5d25f7b551a3ff1cbb Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 20:58:49 -0500 Subject: [PATCH 055/106] comments on dims --- inference/models/mixtral.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 8634550faa..94df4fadeb 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -172,7 +172,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, printf("moe's input, ff_norm, has dims %d %d %d\n", ff_norm->dims[0], ff_norm->dims[1], ff_norm->dims[2]); // printf("moe's input, ff_norm, has shape: %d, %d\n", ff_norm->dims[0], ff_norm->dims[1]); Tensor gate = ff.dense( - ff_norm, + ff_norm, // (hidden_size, 1, 128) mixtral_config.num_local_experts, AC_MODE_NONE, false, @@ -185,14 +185,16 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") .c_str()); - printf("gate has dims %d %d %d\n", gate->dims[0], gate->dims[1], gate->dims[2]); + printf("gate before softmax has dims %d %d %d\n", gate->dims[0], gate->dims[1], gate->dims[2]); gate = ff.softmax( // This operation fails! - gate, + gate, // (num_experts, 1, 128) 0, DT_NONE, std::string("layers." + std::to_string(i) + ".block_sparse_moe_softmax") .c_str()); + printf("gate after softmax has dims %d %d %d\n", gate->dims[0], gate->dims[1], gate->dims[2]); + Tensor topk_out[2] = {nullptr, nullptr}; // (2,) ff.top_k( gate, @@ -201,8 +203,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, false, std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") .c_str()); - Tensor topk_values = topk_out[0]; printf("topk_values has dims %d %d %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2]); - Tensor topk_indices = topk_out[1]; printf("topk_indices has dims %d %d %d\n", topk_indices->dims[0], topk_indices->dims[1], topk_indices->dims[2]); + Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) + Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; ff.group_by( From c0e452481b93c5dacf485cae11bf015d214f0928 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 21:30:29 -0500 Subject: [PATCH 056/106] sync --- inference/models/mixtral.cc | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 94df4fadeb..521a687af3 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -15,6 +15,13 @@ #include "mixtral.h" +//#define NDEBUG +#ifndef NDEBUG +#define dbg_printf(...) printf(__VA_ARGS__) +#else +#define dbg_printf(...) +#endif + namespace FlexFlow { using namespace Legion; @@ -63,7 +70,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor mlp_out = nullptr; for (int i = 0; i < mixtral_config.num_hidden_layers; i++) { - printf("mixtral hidden layer %d\n", i); + dbg_printf("mixtral hidden layer %d\n", i); // set transformer layer id ff.set_transformer_layer_id(i); @@ -169,8 +176,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor ff_norm = token_ff_norm[1]; // MoE - printf("moe's input, ff_norm, has dims %d %d %d\n", ff_norm->dims[0], ff_norm->dims[1], ff_norm->dims[2]); -// printf("moe's input, ff_norm, has shape: %d, %d\n", ff_norm->dims[0], ff_norm->dims[1]); + dbg_printf("moe's input, ff_norm, has dims %d %d %d\n", ff_norm->dims[0], ff_norm->dims[1], ff_norm->dims[2]); +// dbg_printf("moe's input, ff_norm, has shape: %d, %d\n", ff_norm->dims[0], ff_norm->dims[1]); Tensor gate = ff.dense( ff_norm, // (hidden_size, 1, 128) mixtral_config.num_local_experts, @@ -185,7 +192,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") .c_str()); - printf("gate before softmax has dims %d %d %d\n", gate->dims[0], gate->dims[1], gate->dims[2]); + dbg_printf("gate before softmax has dims %d %d %d\n", gate->dims[0], gate->dims[1], gate->dims[2]); gate = ff.softmax( // This operation fails! gate, // (num_experts, 1, 128) 0, @@ -193,12 +200,12 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::string("layers." + std::to_string(i) + ".block_sparse_moe_softmax") .c_str()); - printf("gate after softmax has dims %d %d %d\n", gate->dims[0], gate->dims[1], gate->dims[2]); + dbg_printf("gate after softmax has dims %d %d %d\n", gate->dims[0], gate->dims[1], gate->dims[2]); - Tensor topk_out[2] = {nullptr, nullptr}; // (2,) + Tensor topk_out[2] = {nullptr, nullptr}; ff.top_k( - gate, - topk_out, // (2,) + gate, // (num_experts, 1, 128) + topk_out, mixtral_config.num_experts_per_tok, false, std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") @@ -215,7 +222,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 0.0f, std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby") .c_str()); - printf("grouped_tokens[0] has dims %d %d %d\n", grouped_tokens[0]->dims[0], grouped_tokens[0]->dims[1], grouped_tokens[0]->dims[2]); + + // grouped_tokens[0] has dims (1024, 1, 0) Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; expert_idx++) { Tensor w1 = ff.dense(grouped_tokens[expert_idx], @@ -271,12 +279,12 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, ".block_sparse_moe_experts_" + std::to_string(expert_idx) + "_w2") .c_str()); - printf("w2 and aggreagte_inputs[%d] has dims %d %d %d\n",4 + expert_idx, w2->dims[0], w2->dims[1], w2->dims[2]); + // w2 and aggreagte_inputs[4+] have dims (1024, 1, 0) aggregate_inputs[4 + expert_idx] = w2; } - Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); printf("topk_values_reduced has dims %d %d %d\n", topk_values_reduced->dims[0], topk_values_reduced->dims[1], topk_values_reduced->dims[2]); - topk_values = ff.divide(topk_values, topk_values_reduced); printf("topk_values has dims %d %d %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2]); + Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); dbg_printf("topk_values_reduced has dims %d %d %d\n", topk_values_reduced->dims[0], topk_values_reduced->dims[1], topk_values_reduced->dims[2]); + topk_values = ff.divide(topk_values, topk_values_reduced); dbg_printf("topk_values has dims %d %d %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2]); Tensor dummy_gate = ff.dense( ff_norm, @@ -298,8 +306,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, DT_NONE, std::string("dummy_gate").c_str()); - aggregate_inputs[0] = topk_values; printf("aggregate_inputs[0] has dims %d %d %d\n", aggregate_inputs[0]->dims[0], aggregate_inputs[0]->dims[1], aggregate_inputs[0]->dims[2]); - aggregate_inputs[1] = topk_indices; printf("aggregate_inputs[1] has dims %d %d %d\n", aggregate_inputs[1]->dims[0], aggregate_inputs[1]->dims[1], aggregate_inputs[1]->dims[2]); + aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) + aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) aggregate_inputs[2] = topk_values; // TODO this is a tmp fix aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix // aggregate_inputs[2] = aggregate_inputs[3] = nullptr; From 7462fb4c04ac272f72cb6fcda63e04c3b769e83a Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 21:42:05 -0500 Subject: [PATCH 057/106] sync --- inference/models/mixtral.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 521a687af3..aa8a552df8 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -15,8 +15,8 @@ #include "mixtral.h" -//#define NDEBUG -#ifndef NDEBUG +//#define MIXTRAL_DEBUG +#ifndef MIXTRAL_DEBUG #define dbg_printf(...) printf(__VA_ARGS__) #else #define dbg_printf(...) @@ -280,11 +280,13 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::to_string(expert_idx) + "_w2") .c_str()); // w2 and aggreagte_inputs[4+] have dims (1024, 1, 0) + + printf("w2 has dim count %d\n", w2->num_dims); aggregate_inputs[4 + expert_idx] = w2; } - Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); dbg_printf("topk_values_reduced has dims %d %d %d\n", topk_values_reduced->dims[0], topk_values_reduced->dims[1], topk_values_reduced->dims[2]); - topk_values = ff.divide(topk_values, topk_values_reduced); dbg_printf("topk_values has dims %d %d %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2]); + Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); printf("topk_values_reduced has dims %d %d %d\n", topk_values_reduced->dims[0], topk_values_reduced->dims[1], topk_values_reduced->dims[2]); + topk_values = ff.divide(topk_values, topk_values_reduced); printf("topk_values has dims %d %d %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2]); Tensor dummy_gate = ff.dense( ff_norm, From 21ecf7752b3cee1e604a81af28bbe37179b3b221 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 21:46:41 -0500 Subject: [PATCH 058/106] sync --- inference/models/mixtral.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index aa8a552df8..fb19ee30fb 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -210,6 +210,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, false, std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") .c_str()); + printf("topk_values has dim count %d\n", topk_values->num_dims); Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) @@ -285,8 +286,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[4 + expert_idx] = w2; } - Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); printf("topk_values_reduced has dims %d %d %d\n", topk_values_reduced->dims[0], topk_values_reduced->dims[1], topk_values_reduced->dims[2]); - topk_values = ff.divide(topk_values, topk_values_reduced); printf("topk_values has dims %d %d %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2]); + Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) + topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) Tensor dummy_gate = ff.dense( ff_norm, From 742ec59adb2065ac7060a12d302f24edf9b188c0 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 21:51:24 -0500 Subject: [PATCH 059/106] sync --- inference/models/mixtral.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index fb19ee30fb..cdf43465b9 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -210,9 +210,12 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, false, std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") .c_str()); - printf("topk_values has dim count %d\n", topk_values->num_dims); Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) + printf("topk_values has dim count %d\n", topk_values->num_dims); + printf("topk_values has shape %d %d %d %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2], topk_values->dims[3]); Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) + printf("topk_indices has dim count %d\n", topk_indices->num_dims); + printf("topk_indices has shape %d %d %d %d\n", topk_indices->dims[0], topk_indices->dims[1], topk_indices->dims[2], topk_indices->dims[3]); Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; ff.group_by( From b04af7af1592aa9db98d765aeaa59e8cab47fa5c Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 21:57:41 -0500 Subject: [PATCH 060/106] sync --- inference/models/mixtral.cc | 7 +++---- src/ops/aggregate.cc | 6 ++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index cdf43465b9..d041240f07 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -285,8 +285,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, .c_str()); // w2 and aggreagte_inputs[4+] have dims (1024, 1, 0) - printf("w2 has dim count %d\n", w2->num_dims); - aggregate_inputs[4 + expert_idx] = w2; + aggregate_inputs[4 + expert_idx] = w2; // w2 has 3 dimensions } Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) @@ -312,8 +311,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, DT_NONE, std::string("dummy_gate").c_str()); - aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) - aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) + aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) (3 dims confirmed) + aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) (3 dims confirmed) aggregate_inputs[2] = topk_values; // TODO this is a tmp fix aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix // aggregate_inputs[2] = aggregate_inputs[3] = nullptr; diff --git a/src/ops/aggregate.cc b/src/ops/aggregate.cc index ffa988410c..e0ac0dc2c5 100644 --- a/src/ops/aggregate.cc +++ b/src/ops/aggregate.cc @@ -52,6 +52,9 @@ Tensor FFModel::aggregate( 1 /*outputs*/, inputs); { + + printf("In FFModel::aggregate, inputs[0]->num_dims = %d\n", inputs[0]->num_dims); + int num_dim = inputs[4]->num_dims; // Set output shape int dims[MAX_TENSOR_DIM]; @@ -125,6 +128,7 @@ Aggregate::Aggregate(FFModel &model, assert(n + 4 == numInputs); assert(n > 0); + printf("In Aggregate::Aggregate, inputs[0]->num_dims = %d\n", inputs[0]->num_dims); assert(inputs[0]->num_dims == 2 + 1); assert(inputs[1]->num_dims == 2 + 1); assert(inputs[2]->num_dims == 2 + 1); @@ -263,6 +267,8 @@ void Aggregate::forward(FFModel const &ff) { false /*must*/, 0 /*mapper_id*/, outputs[0]->machine_view.hash()); + + printf("Entered Aggregate::forward\n"); // gate_preds launcher.add_region_requirement(RegionRequirement(inputs[0]->part, 0 /*projection id*/, From 99954e5830fd7bce0576a5e0901b9cdba8577a29 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 22:11:49 -0500 Subject: [PATCH 061/106] sync --- inference/models/mixtral.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index d041240f07..cfbf542902 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -16,7 +16,7 @@ #include "mixtral.h" //#define MIXTRAL_DEBUG -#ifndef MIXTRAL_DEBUG +#ifdef MIXTRAL_DEBUG #define dbg_printf(...) printf(__VA_ARGS__) #else #define dbg_printf(...) From 511fc25bc18578c62a03c86166e50d098da5e04a Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 22:13:00 -0500 Subject: [PATCH 062/106] sync --- inference/models/mixtral.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index cfbf542902..621b42b325 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -176,8 +176,6 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor ff_norm = token_ff_norm[1]; // MoE - dbg_printf("moe's input, ff_norm, has dims %d %d %d\n", ff_norm->dims[0], ff_norm->dims[1], ff_norm->dims[2]); -// dbg_printf("moe's input, ff_norm, has shape: %d, %d\n", ff_norm->dims[0], ff_norm->dims[1]); Tensor gate = ff.dense( ff_norm, // (hidden_size, 1, 128) mixtral_config.num_local_experts, @@ -192,7 +190,6 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") .c_str()); - dbg_printf("gate before softmax has dims %d %d %d\n", gate->dims[0], gate->dims[1], gate->dims[2]); gate = ff.softmax( // This operation fails! gate, // (num_experts, 1, 128) 0, @@ -200,8 +197,6 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::string("layers." + std::to_string(i) + ".block_sparse_moe_softmax") .c_str()); - dbg_printf("gate after softmax has dims %d %d %d\n", gate->dims[0], gate->dims[1], gate->dims[2]); - Tensor topk_out[2] = {nullptr, nullptr}; ff.top_k( gate, // (num_experts, 1, 128) From e590ce55518f62bc51301c155da7869ba5d91268 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 22:59:00 -0500 Subject: [PATCH 063/106] sync --- inference/models/mixtral.cc | 8 ++------ src/ops/aggregate.cc | 11 +++++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 621b42b325..8747ded662 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -205,12 +205,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, false, std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") .c_str()); - Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) - printf("topk_values has dim count %d\n", topk_values->num_dims); - printf("topk_values has shape %d %d %d %d\n", topk_values->dims[0], topk_values->dims[1], topk_values->dims[2], topk_values->dims[3]); - Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) - printf("topk_indices has dim count %d\n", topk_indices->num_dims); - printf("topk_indices has shape %d %d %d %d\n", topk_indices->dims[0], topk_indices->dims[1], topk_indices->dims[2], topk_indices->dims[3]); + Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) (confirmed 3 dims) + Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) (confirmed 3 dims) Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; ff.group_by( diff --git a/src/ops/aggregate.cc b/src/ops/aggregate.cc index e0ac0dc2c5..7cc67cb9f6 100644 --- a/src/ops/aggregate.cc +++ b/src/ops/aggregate.cc @@ -37,6 +37,7 @@ using Legion::TaskArgument; using Legion::TaskLauncher; using PCG::Node; +// This runs when mixtral.cc is run Tensor FFModel::aggregate( Tensor const *inputs, /* gate_preds, gate_assign, gate assign TopK, full_gate_pred, exp_pred_1, ... , exp_pred_n */ @@ -103,6 +104,7 @@ bool operator==(AggregateParams const &lhs, AggregateParams const &rhs) { return lhs.n == rhs.n && lhs.lambda_bal == rhs.lambda_bal; } +// This runs after mixtral.cc is ran and prompt is tokenized Aggregate::Aggregate(FFModel &model, ParallelTensor const *_inputs, int _n, @@ -129,10 +131,11 @@ Aggregate::Aggregate(FFModel &model, assert(n + 4 == numInputs); assert(n > 0); printf("In Aggregate::Aggregate, inputs[0]->num_dims = %d\n", inputs[0]->num_dims); - assert(inputs[0]->num_dims == 2 + 1); - assert(inputs[1]->num_dims == 2 + 1); - assert(inputs[2]->num_dims == 2 + 1); - assert(inputs[3]->num_dims == 2 + 1); + printf("In Aggregate::Aggregate, inputs[0] dims are %d %d %d %d\n", inputs[0]->dims[0], inputs[0]->dims[1], inputs[0]->dims[2], inputs[0]->dims[3]); + assert(inputs[0]->num_dims >= 2 + 1); + assert(inputs[1]->num_dims >= 2 + 1); + assert(inputs[2]->num_dims >= 2 + 1); + assert(inputs[3]->num_dims >= 2 + 1); for (int i = 0; i < inputs[0]->num_dims; i++) { assert(inputs[0]->dims[i] == inputs[1]->dims[i]); From 1ed4bffb0289b4fe0c2411b6b1cfadb05ae39def Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 23:03:00 -0500 Subject: [PATCH 064/106] sync --- src/ops/aggregate.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ops/aggregate.cc b/src/ops/aggregate.cc index 7cc67cb9f6..f4848ab166 100644 --- a/src/ops/aggregate.cc +++ b/src/ops/aggregate.cc @@ -131,7 +131,7 @@ Aggregate::Aggregate(FFModel &model, assert(n + 4 == numInputs); assert(n > 0); printf("In Aggregate::Aggregate, inputs[0]->num_dims = %d\n", inputs[0]->num_dims); - printf("In Aggregate::Aggregate, inputs[0] dims are %d %d %d %d\n", inputs[0]->dims[0], inputs[0]->dims[1], inputs[0]->dims[2], inputs[0]->dims[3]); + printf("In Aggregate::Aggregate, inputs[0] dims are %d %d %d %d\n", inputs[0]->dims[0].size, inputs[0]->dims[1].size, inputs[0]->dims[2].size, inputs[0]->dims[3].size); assert(inputs[0]->num_dims >= 2 + 1); assert(inputs[1]->num_dims >= 2 + 1); assert(inputs[2]->num_dims >= 2 + 1); From 570037897d481fcf849a9f735a966b9a88195e81 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 23:36:15 -0500 Subject: [PATCH 065/106] CHECKPOINT I'm currently able to tokenize prompt and read through mixtral.cc. However, I get a op->numInputs = 1 assertion error in aggregate.cc for the aggregation in the last layer. I was previously able to output tokens, but with only one expert (I changed mixtral.cc to have only one expert and no aggregation). --- src/ops/aggregate.cc | 15 ++++++++------- src/runtime/graph.cc | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/ops/aggregate.cc b/src/ops/aggregate.cc index f4848ab166..089dbe8be8 100644 --- a/src/ops/aggregate.cc +++ b/src/ops/aggregate.cc @@ -104,7 +104,7 @@ bool operator==(AggregateParams const &lhs, AggregateParams const &rhs) { return lhs.n == rhs.n && lhs.lambda_bal == rhs.lambda_bal; } -// This runs after mixtral.cc is ran and prompt is tokenized +// This runs after mixtral.cc is ran and the prompt is tokenized Aggregate::Aggregate(FFModel &model, ParallelTensor const *_inputs, int _n, @@ -114,9 +114,9 @@ Aggregate::Aggregate(FFModel &model, OP_AGGREGATE, DT_FLOAT, name, - _n + 4 /*inputs*/, - 0 /*weights*/, - 1 /*outputs*/, + _n + 4 /*numInputs*/, + 0 /*numWeights*/, + 1 /*numOutputs*/, _inputs), n(_n), lambda_bal(_lambda_bal) { // FIXME: For now, set upper limits Better: Do as follows, but memory is @@ -130,9 +130,10 @@ Aggregate::Aggregate(FFModel &model, assert(n + 4 == numInputs); assert(n > 0); - printf("In Aggregate::Aggregate, inputs[0]->num_dims = %d\n", inputs[0]->num_dims); - printf("In Aggregate::Aggregate, inputs[0] dims are %d %d %d %d\n", inputs[0]->dims[0].size, inputs[0]->dims[1].size, inputs[0]->dims[2].size, inputs[0]->dims[3].size); - assert(inputs[0]->num_dims >= 2 + 1); + //printf("In Aggregate::Aggregate, inputs[0]->num_dims = %d\n", inputs[0]->num_dims); + //printf("In Aggregate::Aggregate, inputs[0] dims are %d %d %d %d\n", inputs[0]->dims[0].size, inputs[0]->dims[1].size, inputs[0]->dims[2].size, inputs[0]->dims[3].size); + // TODO the inequalities below used to be equalities, not sure it's a good idea to switch to inequalities + assert(inputs[0]->num_dims >= 2 + 1); // inputs[0] has dims (experts_per_token, 1, 128, 1) (confirmed dim count) assert(inputs[1]->num_dims >= 2 + 1); assert(inputs[2]->num_dims >= 2 + 1); assert(inputs[3]->num_dims >= 2 + 1); diff --git a/src/runtime/graph.cc b/src/runtime/graph.cc index 2bc64c1670..6efea07048 100644 --- a/src/runtime/graph.cc +++ b/src/runtime/graph.cc @@ -2125,7 +2125,7 @@ GraphOptimalViewSerialized // Be optimistic lambdas.emplace_back(std::make_pair(1.0, MemorySearchResult{})); auto try_result = try_one_lambda( - lambdas.back(), task, cached_simulator, perform_memory_search); + lambdas.back(), task, cached_simulator, perform_memory_search); best_graph = std::move(try_result.first); optimal_views = try_result.second; From 381d3cd0c46337571bada9762c6c962cf7f6ff0e Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sat, 7 Dec 2024 23:47:47 -0500 Subject: [PATCH 066/106] 2222:22 port --- docker/run-persistent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/run-persistent.sh b/docker/run-persistent.sh index 162ba72795..2b746c7026 100755 --- a/docker/run-persistent.sh +++ b/docker/run-persistent.sh @@ -129,6 +129,6 @@ if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" fi -docker_command="docker run -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${cache_volume} ${home_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" +docker_command="docker run -it -p 2222:22 $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${cache_volume} ${home_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" echo "$docker_command" eval "$docker_command" \ No newline at end of file From 6309e70b11090246308d706527d27b001a25b284 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 01:02:04 -0500 Subject: [PATCH 067/106] tmp_volume --- .gitignore | 3 ++- docker/run-persistent.sh | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 4866ebbc90..c96287ca54 100644 --- a/.gitignore +++ b/.gitignore @@ -199,4 +199,5 @@ fine_grained_alignment_config.json # CLion .idea/ -cmake-build-debug \ No newline at end of file +cmake-build-debug +cmake-build-debug-remote-host diff --git a/docker/run-persistent.sh b/docker/run-persistent.sh index 2b746c7026..ecf0a71f62 100755 --- a/docker/run-persistent.sh +++ b/docker/run-persistent.sh @@ -118,10 +118,10 @@ if [[ "$(docker images -q "${image}-${FF_GPU_BACKEND}${gpu_backend_version}":lat exit 1 fi -#cache_volume="-v ${HOME}/.cache:/root/.cache" -#home_volume="-v ${HOME}/dockerhome:/home" cache_volume="-v cache_volume:/root/.cache" home_volume="-v home_volume:/home" +tmp_volume="-v tmp_volume:/tmp" + ssh_key_volume="" ssh_key_path="$HOME/.ssh/id_rsa" @@ -129,6 +129,6 @@ if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" fi -docker_command="docker run -it -p 2222:22 $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${cache_volume} ${home_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" +docker_command="docker run -it -p 2222:22 $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${cache_volume} ${home_volume} ${tmp_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" echo "$docker_command" eval "$docker_command" \ No newline at end of file From 11a78f25dd9872f8238e51d3b8c6a85f1bd8eb8e Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 10:32:31 -0500 Subject: [PATCH 068/106] revert changes --- docker/run-bind.sh | 135 ------------------------------------ inference/models/llama.cc | 60 ++++++++++++++++ inference/models/mixtral.cc | 44 ++++++++++++ 3 files changed, 104 insertions(+), 135 deletions(-) delete mode 100644 docker/run-bind.sh diff --git a/docker/run-bind.sh b/docker/run-bind.sh deleted file mode 100644 index c24b8768cf..0000000000 --- a/docker/run-bind.sh +++ /dev/null @@ -1,135 +0,0 @@ -#! /usr/bin/env bash -set -euo pipefail - -# Usage: ./run.sh -# Optional environment variables: FF_GPU_BACKEND, cuda_version, hip_version, ATTACH_GPUS, SHM_SIZE - -# Cd into directory holding this script -cd "${BASH_SOURCE[0]%/*}" - -# Parse input params -image=${1:-flexflow} -FF_GPU_BACKEND=${FF_GPU_BACKEND:-cuda} -cuda_version=${cuda_version:-"empty"} -hip_version=${hip_version:-"empty"} - -# Parameter controlling whether to attach GPUs to the Docker container -ATTACH_GPUS=${ATTACH_GPUS:-true} -gpu_arg="" -if $ATTACH_GPUS ; then gpu_arg="--gpus all" ; fi -FORWARD_STREAMLIT_PORT=${FORWARD_STREAMLIT_PORT:-true} -port_forward_arg="" -if $FORWARD_STREAMLIT_PORT ; then - port_forward_arg+="-p 8501:8501" -fi - - -# Amount of shared memory to give the Docker container access to -# If you get a Bus Error, increase this value. If you don't have enough memory -# on your machine, decrease this value. -SHM_SIZE=${SHM_SIZE:-8192m} - -# Check docker image name -if [[ "$image" != @(flexflow-environment|flexflow) ]]; then - echo "Error, image name ${image} is invalid. Choose between 'flexflow-environment', 'flexflow'." - exit 1 -fi - -# Check GPU backend -if [[ "${FF_GPU_BACKEND}" != @(cuda|hip_cuda|hip_rocm|intel) ]]; then - echo "Error, value of FF_GPU_BACKEND (${FF_GPU_BACKEND}) is invalid. Pick between 'cuda', 'hip_cuda', 'hip_rocm' or 'intel'." - exit 1 -elif [[ "${FF_GPU_BACKEND}" != "cuda" ]]; then - echo "Running $image docker image with gpu backend: ${FF_GPU_BACKEND}" -else - echo "Running $image docker image with default GPU backend: cuda" -fi - -# gpu backend version suffix for the docker image. -gpu_backend_version="" - -if [[ "${FF_GPU_BACKEND}" == "cuda" || "${FF_GPU_BACKEND}" == "hip_cuda" ]]; then - # Autodetect cuda version if not specified - if [[ $cuda_version == "empty" ]]; then - # shellcheck disable=SC2015 - cuda_version=$(command -v nvcc >/dev/null 2>&1 && nvcc --version | grep "release" | awk '{print $NF}' || true) - # Change cuda_version eg. V11.7.99 to 11.7 - cuda_version=${cuda_version:1:4} - if [[ -z "$cuda_version" ]]; then - echo "Could not detect CUDA version. Please specify one manually by setting the 'cuda_version' env." - exit 1 - fi - fi - # Check that CUDA version is supported - if [[ "$cuda_version" != @(11.1|11.2|11.3|11.4|11.5|11.6|11.7|11.8|12.0|12.1|12.2|12.3|12.4|12.5|12.6|12.7|12.8|12.9) ]]; then - echo "cuda_version is not supported, please choose among {11.1|11.2|11.3|11.4|11.5|11.6|11.7|11.8|12.0|12.1|12.2}" - exit 1 - fi - # Use CUDA 12.2 for all versions greater or equal to 12.2 for now - if [[ "$cuda_version" == @(12.3|12.4|12.5|12.6|12.7|12.8|12.9) ]]; then - cuda_version=12.2 - fi - # Set cuda version suffix to docker image name - echo "Running $image docker image with CUDA $cuda_version" - gpu_backend_version="-${cuda_version}" -fi - -if [[ "${FF_GPU_BACKEND}" == "hip_rocm" || "${FF_GPU_BACKEND}" == "hip_cuda" ]]; then - # Autodetect HIP version if not specified - if [[ $hip_version == "empty" ]]; then - # shellcheck disable=SC2015 - hip_version=$(command -v hipcc >/dev/null 2>&1 && hipcc --version | grep "HIP version:" | awk '{print $NF}' || true) - # Change hip_version eg. 5.6.31061-8c743ae5d to 5.6 - hip_version=${hip_version:0:3} - if [[ -z "$hip_version" ]]; then - echo "Could not detect HIP version. Please specify one manually by setting the 'hip_version' env." - exit 1 - fi - fi - # Check that HIP version is supported - if [[ "$hip_version" != @(5.3|5.4|5.5|5.6) ]]; then - echo "hip_version is not supported, please choose among {5.3, 5.4, 5.5, 5.6}" - exit 1 - fi - echo "Running $image docker image with HIP $hip_version" - if [[ "${FF_GPU_BACKEND}" == "hip_rocm" ]]; then - gpu_backend_version="-${hip_version}" - fi -fi - -# Check that image exists, if fails, print the default error message. -if [[ "$(docker images -q "${image}-${FF_GPU_BACKEND}${gpu_backend_version}":latest 2> /dev/null)" == "" ]]; then - echo "Error, ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest does not exist!" - if [[ "${FF_GPU_BACKEND}" == "cuda" ]]; then - echo "" - echo "To download the docker image, run:" - echo " FF_GPU_BACKEND=${FF_GPU_BACKEND} cuda_version=${cuda_version} $(pwd)/pull.sh $image" - echo "To build the docker image from source, run:" - echo " FF_GPU_BACKEND=${FF_GPU_BACKEND} cuda_version=${cuda_version} $(pwd)/build.sh $image" - echo "" - elif [[ "${FF_GPU_BACKEND}" == "hip_rocm" ]]; then - echo "" - echo "To download the docker image, run:" - echo " FF_GPU_BACKEND=${FF_GPU_BACKEND} hip_version=${hip_version} $(pwd)/pull.sh $image" - echo "To build the docker image from source, run:" - echo " FF_GPU_BACKEND=${FF_GPU_BACKEND} hip_version=${hip_version} $(pwd)/build.sh $image" - echo "" - fi - exit 1 -fi - -hf_token_volume="" -hf_token_path="$HOME/.cache/huggingface/token" -if [ -f "$hf_token_path" ]; then - # If the token exists, add the volume mount to the Docker command - hf_token_volume+="-v $hf_token_path:/root/.cache/huggingface/token" -fi - -ssh_key_volume="" -ssh_key_path="$HOME/.ssh/id_rsa" -if [ -f "$ssh_key_path" ] && [ -f "$ssh_key_path.pub" ]; then - ssh_key_volume="-v $ssh_key_path:/root/.ssh/id_rsa -v $ssh_key_path.pub:/root/.ssh/id_rsa.pub" -fi -docker_command="docker run -v my-volume:/home -it $gpu_arg --shm-size=${SHM_SIZE} --cap-add=SYS_PTRACE ${ssh_key_volume} ${hf_token_volume} ${port_forward_arg} ${image}-${FF_GPU_BACKEND}${gpu_backend_version}:latest" -echo "$docker_command" -eval "$docker_command" \ No newline at end of file diff --git a/inference/models/llama.cc b/inference/models/llama.cc index 157cd7c158..7b4a14b472 100644 --- a/inference/models/llama.cc +++ b/inference/models/llama.cc @@ -109,6 +109,50 @@ void LLAMA::create_llama_model(FFModel &ff, Tensor mha; switch (mode) { + case BEAM_SEARCH_MODE: { + mha = ff.spec_inc_multiquery_self_attention( + qkv_proj, + llama_config.hidden_size, + llama_config.num_attention_heads, + llama_config.num_key_value_heads, + llama_config.hidden_size / llama_config.num_attention_heads, + llama_config.hidden_size / llama_config.num_attention_heads, + 0.0f, /*dropout*/ + false, /*add_zero_attn*/ + DT_NONE, /*data_type*/ + NULL, /*kernel_initializer*/ + llama_config.rotary_embedding_meta, + false, /*scaling query*/ + 1.0f, /*scaling factor*/ + true, /*qk_prod_scaling*/ + false, /*position_bias*/ + std::string("layers." + std::to_string(i) + ".self_attn") + .c_str() /*name*/ + ); + break; + } + case TREE_VERIFY_MODE: { + mha = ff.inc_multiquery_self_attention_verify( + qkv_proj, + llama_config.hidden_size, + llama_config.num_attention_heads, + llama_config.num_key_value_heads, + llama_config.hidden_size / llama_config.num_attention_heads, + llama_config.hidden_size / llama_config.num_attention_heads, + 0.0f, /*dropout*/ + false, /*add_zero_attn*/ + DT_NONE, /*data_type*/ + nullptr, /*kernel_initializer*/ + llama_config.rotary_embedding_meta, + false, /*scaling query*/ + 1.0f, /*scaling factor*/ + true, /*qk_prod_scaling*/ + false, /*position_bias*/ + std::string("layers." + std::to_string(i) + ".self_attn") + .c_str() /*name*/ + ); + break; + } case INC_DECODING_MODE: { mha = ff.inc_multiquery_self_attention( qkv_proj, @@ -231,6 +275,13 @@ void LLAMA::create_llama_model(FFModel &ff, "lm_head"); Tensor output; + if (mode == BEAM_SEARCH_MODE) { + Tensor softmax = ff.softmax(dense, -1); + // output = ff.beam_top_k(softmax, llama_config.max_beam_width, false); + // output = ff.argmax(softmax, /*beam_Search*/ true); + output = ff.arg_top_k(softmax, llama_config.max_beam_width, false, true); + // output = ff.top_k(softmax, ) + } else { // Tensor softmax = ff.softmax(dense, -1); if (generation_config.do_sample) { dense = ff.scalar_truediv(dense, generation_config.temperature, false); @@ -241,6 +292,15 @@ void LLAMA::create_llama_model(FFModel &ff, Tensor softmax = ff.softmax(dense, -1); output = ff.argmax(softmax, /*beam_Search*/ false); } + } + + // If PEFT is enabled, add LoRA layers + if (ff.config.enable_peft) { + // todo: add attention projections + std::vector target_modules = { + "gate_proj", "up_proj", "down_proj"}; + ff.add_lora_layers(target_modules); + } FileDataLoader *fileloader = new FileDataLoader( "", diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 8747ded662..caef252bb6 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -118,6 +118,50 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor mha; switch (mode) { + case BEAM_SEARCH_MODE: { + mha = ff.spec_inc_multiquery_self_attention( + qkv_proj, + mixtral_config.hidden_size, + mixtral_config.num_attention_heads, + mixtral_config.num_key_value_heads, + mixtral_config.hidden_size / mixtral_config.num_attention_heads, + mixtral_config.hidden_size / mixtral_config.num_attention_heads, + 0.0f, /*dropout*/ + false, /*add_zero_attn*/ + DT_NONE, /*data_type*/ + NULL, /*kernel_initializer*/ + mixtral_config.rotary_embedding_meta, + false, /*scaling query*/ + 1.0f, /*scaling factor*/ + true, /*qk_prod_scaling*/ + false, /*position_bias*/ + std::string("layers." + std::to_string(i) + ".self_attn") + .c_str() /*name*/ + ); + break; + } + case TREE_VERIFY_MODE: { + mha = ff.inc_multiquery_self_attention_verify( + qkv_proj, + mixtral_config.hidden_size, + mixtral_config.num_attention_heads, + mixtral_config.num_key_value_heads, + mixtral_config.hidden_size / mixtral_config.num_attention_heads, + mixtral_config.hidden_size / mixtral_config.num_attention_heads, + 0.0f, /*dropout*/ + false, /*add_zero_attn*/ + DT_NONE, /*data_type*/ + nullptr, /*kernel_initializer*/ + mixtral_config.rotary_embedding_meta, + false, /*scaling query*/ + 1.0f, /*scaling factor*/ + true, /*qk_prod_scaling*/ + false, /*position_bias*/ + std::string("layers." + std::to_string(i) + ".self_attn") + .c_str() /*name*/ + ); + break; + } case INC_DECODING_MODE: { mha = ff.inc_multiquery_self_attention( qkv_proj, From 7069082741c5e8e2b05681db7141994389791c0c Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 11:37:12 -0500 Subject: [PATCH 069/106] printf dims --- inference/models/mixtral.cc | 3 ++ src/ops/aggregate.cc | 65 ++++++++++++++++++++++--------------- 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index caef252bb6..b0d010bcc6 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -264,6 +264,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, // grouped_tokens[0] has dims (1024, 1, 0) Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; + Tensor one_aggregate_inputs[1] = {nullptr}; for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; expert_idx++) { Tensor w1 = ff.dense(grouped_tokens[expert_idx], mixtral_config.intermediate_size, @@ -321,6 +322,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, // w2 and aggreagte_inputs[4+] have dims (1024, 1, 0) aggregate_inputs[4 + expert_idx] = w2; // w2 has 3 dimensions + one_aggregate_inputs[0] = w2; } Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) @@ -352,6 +354,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix // aggregate_inputs[2] = aggregate_inputs[3] = nullptr; mlp_out = ff.aggregate(aggregate_inputs, +// topk_values->dims[2], mixtral_config.num_local_experts, 0.0f, std::string("layers." + std::to_string(i) + diff --git a/src/ops/aggregate.cc b/src/ops/aggregate.cc index 089dbe8be8..0e840abdc4 100644 --- a/src/ops/aggregate.cc +++ b/src/ops/aggregate.cc @@ -122,36 +122,47 @@ Aggregate::Aggregate(FFModel &model, // FIXME: For now, set upper limits Better: Do as follows, but memory is // assigned per block, so requires to check that // https://stackoverflow.com/questions/5531247/allocating-shared-memory/5531640#5531640 - assert(n <= AGGREGATE_MAX_N && "Increase AGGREGATE_MAX_N in #define"); - assert(inputs[0]->dims[0].size <= AGGREGATE_MAX_K && - "Increase AGGREGATE_MAX_K in #define"); - assert(inputs[0]->dims[1].size <= AGGREGATE_MAX_BATCH_SIZE && - "Increase AGGREGATE_MAX_BATCH_SIZE in #define"); - - assert(n + 4 == numInputs); - assert(n > 0); - //printf("In Aggregate::Aggregate, inputs[0]->num_dims = %d\n", inputs[0]->num_dims); - //printf("In Aggregate::Aggregate, inputs[0] dims are %d %d %d %d\n", inputs[0]->dims[0].size, inputs[0]->dims[1].size, inputs[0]->dims[2].size, inputs[0]->dims[3].size); - // TODO the inequalities below used to be equalities, not sure it's a good idea to switch to inequalities - assert(inputs[0]->num_dims >= 2 + 1); // inputs[0] has dims (experts_per_token, 1, 128, 1) (confirmed dim count) - assert(inputs[1]->num_dims >= 2 + 1); - assert(inputs[2]->num_dims >= 2 + 1); - assert(inputs[3]->num_dims >= 2 + 1); - - for (int i = 0; i < inputs[0]->num_dims; i++) { - assert(inputs[0]->dims[i] == inputs[1]->dims[i]); - assert(inputs[0]->dims[i] == inputs[2]->dims[i]); - } - assert(inputs[0]->dims[1] == inputs[3]->dims[1]); - assert(inputs[3]->dims[0].size == n); + + printf('_inputs[0]->dims[1].size = %d\n', _inputs[0]->dims[1].size); + printf('_inputs[0]->dims[1].degree = %d\n', _inputs[0]->dims[1].degree); + printf('_inputs[0]->dims[1].parallel_idx = %d\n', _inputs[0]->dims[1].parallel_idx); + printf('_inputs[0]->dims[1].is_replica_dim = %d\n', _inputs[0]->dims[1].is_replica_dim); + + printf('_inputs[0]->dims[2].size = %d\n', _inputs[0]->dims[2].size); + printf('_inputs[0]->dims[2].degree = %d\n', _inputs[0]->dims[2].degree); + printf('_inputs[0]->dims[2].parallel_idx = %d\n', _inputs[0]->dims[2].parallel_idx); + printf('_inputs[0]->dims[2].is_replica_dim = %d\n', _inputs[0]->dims[2].is_replica_dim); + +// assert(n <= AGGREGATE_MAX_N && "Increase AGGREGATE_MAX_N in #define"); +// assert(inputs[0]->dims[0].size <= AGGREGATE_MAX_K && +// "Increase AGGREGATE_MAX_K in #define"); +// assert(inputs[0]->dims[1].size <= AGGREGATE_MAX_BATCH_SIZE && +// "Increase AGGREGATE_MAX_BATCH_SIZE in #define"); +// +// assert(n + 4 == numInputs); +// assert(n > 0); +// //printf("In Aggregate::Aggregate, inputs[0]->num_dims = %d\n", inputs[0]->num_dims); +// //printf("In Aggregate::Aggregate, inputs[0] dims are %d %d %d %d\n", inputs[0]->dims[0].size, inputs[0]->dims[1].size, inputs[0]->dims[2].size, inputs[0]->dims[3].size); +// // TODO the inequalities below used to be equalities, not sure it's a good idea to switch to inequalities +// assert(inputs[0]->num_dims >= 2 + 1); // inputs[0] has dims (experts_per_token, 1, 128, 1) (confirmed dim count) +// assert(inputs[1]->num_dims >= 2 + 1); +// assert(inputs[2]->num_dims >= 2 + 1); +// assert(inputs[3]->num_dims >= 2 + 1); +// +// for (int i = 0; i < inputs[0]->num_dims; i++) { +// assert(inputs[0]->dims[i] == inputs[1]->dims[i]); +// assert(inputs[0]->dims[i] == inputs[2]->dims[i]); +// } +// assert(inputs[0]->dims[1] == inputs[3]->dims[1]); +// assert(inputs[3]->dims[0].size == n); // expert inputs - int num_dim = inputs[4]->num_dims; + int num_dim = inputs[4]->num_dims; // 3 int out_dim = inputs[4]->dims[0].size; - for (int i = 1; i < n; i++) { - assert(inputs[i + 4]->num_dims == num_dim); - assert(inputs[i + 4]->dims[0].size == out_dim); - } +// for (int i = 1; i < n; i++) { +// assert(inputs[i + 4]->num_dims == num_dim); +// assert(inputs[i + 4]->dims[0].size == out_dim); +// } // Set output shape ParallelDim dims[MAX_TENSOR_DIM]; for (int i = 0; i < num_dim - 1; i++) { From 4efc9ebf2a2ba7402fa2d8b535665f6266ced534 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 11:39:48 -0500 Subject: [PATCH 070/106] sync --- src/ops/aggregate.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/ops/aggregate.cc b/src/ops/aggregate.cc index 0e840abdc4..8783d8f210 100644 --- a/src/ops/aggregate.cc +++ b/src/ops/aggregate.cc @@ -123,15 +123,15 @@ Aggregate::Aggregate(FFModel &model, // assigned per block, so requires to check that // https://stackoverflow.com/questions/5531247/allocating-shared-memory/5531640#5531640 - printf('_inputs[0]->dims[1].size = %d\n', _inputs[0]->dims[1].size); - printf('_inputs[0]->dims[1].degree = %d\n', _inputs[0]->dims[1].degree); - printf('_inputs[0]->dims[1].parallel_idx = %d\n', _inputs[0]->dims[1].parallel_idx); - printf('_inputs[0]->dims[1].is_replica_dim = %d\n', _inputs[0]->dims[1].is_replica_dim); - - printf('_inputs[0]->dims[2].size = %d\n', _inputs[0]->dims[2].size); - printf('_inputs[0]->dims[2].degree = %d\n', _inputs[0]->dims[2].degree); - printf('_inputs[0]->dims[2].parallel_idx = %d\n', _inputs[0]->dims[2].parallel_idx); - printf('_inputs[0]->dims[2].is_replica_dim = %d\n', _inputs[0]->dims[2].is_replica_dim); + printf("_inputs[0]->dims[1].size = %d\n", _inputs[0]->dims[1].size); + printf("_inputs[0]->dims[1].degree = %d\n", _inputs[0]->dims[1].degree); + printf("_inputs[0]->dims[1].parallel_idx = %d\n", _inputs[0]->dims[1].parallel_idx); + printf("_inputs[0]->dims[1].is_replica_dim = %d\n", _inputs[0]->dims[1].is_replica_dim); + + printf("_inputs[0]->dims[2].size = %d\n", _inputs[0]->dims[2].size); + printf("_inputs[0]->dims[2].degree = %d\n", _inputs[0]->dims[2].degree); + printf("_inputs[0]->dims[2].parallel_idx = %d\n", _inputs[0]->dims[2].parallel_idx); + printf("_inputs[0]->dims[2].is_replica_dim = %d\n", _inputs[0]->dims[2].is_replica_dim); // assert(n <= AGGREGATE_MAX_N && "Increase AGGREGATE_MAX_N in #define"); // assert(inputs[0]->dims[0].size <= AGGREGATE_MAX_K && From fe1f6f25ee4d59d0ae6fea94307962caa66f398d Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 13:39:18 -0500 Subject: [PATCH 071/106] sync --- inference/models/mixtral.cc | 16 +++++++++------- src/ops/aggregate.cc | 19 +++++++++++++++++-- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index b0d010bcc6..85d49802e9 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -353,13 +353,15 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[2] = topk_values; // TODO this is a tmp fix aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix // aggregate_inputs[2] = aggregate_inputs[3] = nullptr; - mlp_out = ff.aggregate(aggregate_inputs, -// topk_values->dims[2], - mixtral_config.num_local_experts, - 0.0f, - std::string("layers." + std::to_string(i) + - ".block_sparse_moe_experts_aggregate") - .c_str()); + + mlp_out = aggregate_inputs[5]; +// mlp_out = ff.aggregate(aggregate_inputs, +//// topk_values->dims[2], +// mixtral_config.num_local_experts, +// 0.0f, +// std::string("layers." + std::to_string(i) + +// ".block_sparse_moe_experts_aggregate") +// .c_str()); } // final normalization and linear diff --git a/src/ops/aggregate.cc b/src/ops/aggregate.cc index 8783d8f210..3440a81533 100644 --- a/src/ops/aggregate.cc +++ b/src/ops/aggregate.cc @@ -168,8 +168,23 @@ Aggregate::Aggregate(FFModel &model, for (int i = 0; i < num_dim - 1; i++) { dims[i] = inputs[4]->dims[i]; } - dims[num_dim - 2] = inputs[0]->dims[num_dim - 2]; - dims[num_dim - 1] = inputs[0]->dims[num_dim - 1]; + + // TODO replace with inputs[0]->dims[num_dim - 2] + ParallelDim topk_values_penultimate_dim; + topk_values_penultimate_dim.size = 1; + topk_values_penultimate_dim.degree = 1; + topk_values_penultimate_dim.parallel_idx = -1; + topk_values_penultimate_dim.is_replica_dim = false; + + // TODO replace with inputs[0]->dims[num_dim - 1] + ParallelDim topk_values_last_dim; + topk_values_last_dim.size = 128; + topk_values_last_dim.degree = 1; + topk_values_last_dim.parallel_idx = -1; + topk_values_last_dim.is_replica_dim = false; + + dims[num_dim - 2] = topk_values_penultimate_dim; + dims[num_dim - 1] = topk_values_last_dim; numOutputs = 1; outputs[0] = model.create_parallel_tensor_legion_ordering( num_dim, dims, DT_FLOAT, this); From d71e6740bd397f0f166a7e4261f3d771094b6c6f Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 13:46:40 -0500 Subject: [PATCH 072/106] sync --- inference/models/mixtral.cc | 234 ++++++------------------------------ 1 file changed, 39 insertions(+), 195 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 85d49802e9..2691c72040 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -15,13 +15,6 @@ #include "mixtral.h" -//#define MIXTRAL_DEBUG -#ifdef MIXTRAL_DEBUG -#define dbg_printf(...) printf(__VA_ARGS__) -#else -#define dbg_printf(...) -#endif - namespace FlexFlow { using namespace Legion; @@ -70,8 +63,6 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor mlp_out = nullptr; for (int i = 0; i < mixtral_config.num_hidden_layers; i++) { - dbg_printf("mixtral hidden layer %d\n", i); - // set transformer layer id ff.set_transformer_layer_id(i); @@ -118,50 +109,6 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor mha; switch (mode) { - case BEAM_SEARCH_MODE: { - mha = ff.spec_inc_multiquery_self_attention( - qkv_proj, - mixtral_config.hidden_size, - mixtral_config.num_attention_heads, - mixtral_config.num_key_value_heads, - mixtral_config.hidden_size / mixtral_config.num_attention_heads, - mixtral_config.hidden_size / mixtral_config.num_attention_heads, - 0.0f, /*dropout*/ - false, /*add_zero_attn*/ - DT_NONE, /*data_type*/ - NULL, /*kernel_initializer*/ - mixtral_config.rotary_embedding_meta, - false, /*scaling query*/ - 1.0f, /*scaling factor*/ - true, /*qk_prod_scaling*/ - false, /*position_bias*/ - std::string("layers." + std::to_string(i) + ".self_attn") - .c_str() /*name*/ - ); - break; - } - case TREE_VERIFY_MODE: { - mha = ff.inc_multiquery_self_attention_verify( - qkv_proj, - mixtral_config.hidden_size, - mixtral_config.num_attention_heads, - mixtral_config.num_key_value_heads, - mixtral_config.hidden_size / mixtral_config.num_attention_heads, - mixtral_config.hidden_size / mixtral_config.num_attention_heads, - 0.0f, /*dropout*/ - false, /*add_zero_attn*/ - DT_NONE, /*data_type*/ - nullptr, /*kernel_initializer*/ - mixtral_config.rotary_embedding_meta, - false, /*scaling query*/ - 1.0f, /*scaling factor*/ - true, /*qk_prod_scaling*/ - false, /*position_bias*/ - std::string("layers." + std::to_string(i) + ".self_attn") - .c_str() /*name*/ - ); - break; - } case INC_DECODING_MODE: { mha = ff.inc_multiquery_self_attention( qkv_proj, @@ -220,150 +167,47 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor ff_norm = token_ff_norm[1]; // MoE - Tensor gate = ff.dense( - ff_norm, // (hidden_size, 1, 128) - mixtral_config.num_local_experts, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") - .c_str()); - - gate = ff.softmax( // This operation fails! - gate, // (num_experts, 1, 128) - 0, - DT_NONE, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_softmax") - .c_str()); - - Tensor topk_out[2] = {nullptr, nullptr}; - ff.top_k( - gate, // (num_experts, 1, 128) - topk_out, - mixtral_config.num_experts_per_tok, - false, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") - .c_str()); - Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) (confirmed 3 dims) - Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) (confirmed 3 dims) - - Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; - ff.group_by( - ff_norm, - topk_indices, - grouped_tokens, - mixtral_config.num_local_experts, - 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby") - .c_str()); - - // grouped_tokens[0] has dims (1024, 1, 0) - Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; - Tensor one_aggregate_inputs[1] = {nullptr}; - for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; expert_idx++) { - Tensor w1 = ff.dense(grouped_tokens[expert_idx], - mixtral_config.intermediate_size, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers." + std::to_string(i) + - ".block_sparse_moe_experts_" + - std::to_string(expert_idx) + "_w1") - .c_str()); - - Tensor w3 = ff.dense(grouped_tokens[expert_idx], - mixtral_config.intermediate_size, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers." + std::to_string(i) + - ".block_sparse_moe_experts_" + - std::to_string(expert_idx) + "_w3") - .c_str()); - - Tensor multi = - ff.sigmoid_silu_multi(w1, - w3, - DT_NONE, - std::string("layers." + std::to_string(i) + - ".block_sparse_moe_experts_" + - std::to_string(expert_idx) + "ssm") - .c_str()); - - Tensor w2 = ff.dense(multi, - mixtral_config.hidden_size, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers." + std::to_string(i) + - ".block_sparse_moe_experts_" + - std::to_string(expert_idx) + "_w2") - .c_str()); - // w2 and aggreagte_inputs[4+] have dims (1024, 1, 0) - - aggregate_inputs[4 + expert_idx] = w2; // w2 has 3 dimensions - one_aggregate_inputs[0] = w2; - } - - Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) - topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) - - Tensor dummy_gate = ff.dense( - ff_norm, - mixtral_config.num_local_experts, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") - .c_str()); - - dummy_gate = ff.softmax( - gate, - 0, - DT_NONE, - std::string("dummy_gate").c_str()); + Tensor w1 = ff.dense( + ff_norm, + mixtral_config.intermediate_size, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w1").c_str()); + + Tensor w3 = ff.dense( + ff_norm, + mixtral_config.intermediate_size, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w3").c_str()); - aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) (3 dims confirmed) - aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) (3 dims confirmed) - aggregate_inputs[2] = topk_values; // TODO this is a tmp fix - aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix -// aggregate_inputs[2] = aggregate_inputs[3] = nullptr; + Tensor multi = ff.sigmoid_silu_multi(w1, w3); //DT_NONE,std::string("layers." + std::to_string(i) +".block_sparse_moe_experts." +std::to_string(expert_idx) + "ssm").c_str()); - mlp_out = aggregate_inputs[5]; -// mlp_out = ff.aggregate(aggregate_inputs, -//// topk_values->dims[2], -// mixtral_config.num_local_experts, -// 0.0f, -// std::string("layers." + std::to_string(i) + -// ".block_sparse_moe_experts_aggregate") -// .c_str()); + mlp_out = ff.dense( + multi, + mixtral_config.hidden_size, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w2").c_str()); } - // final normalization and linear Tensor final_rms_norm_output[2] = {nullptr, nullptr}; ff.residual_rms_norm(token, @@ -412,4 +256,4 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, im->register_model_weights_loader(&ff, fileloader); } -}; // namespace FlexFlow +}; // namespace FlexFlow \ No newline at end of file From 3ae5a8582c1f76cb23cf2fe6c486c6e3303cbe0c Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 13:54:23 -0500 Subject: [PATCH 073/106] sync --- inference/models/mixtral.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 2691c72040..ccb66a44cc 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -60,6 +60,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, embed_init, "embed_tokens"); + printf("token before iteration dims are %d %d %d %d\n", token->dims[0].size, token->dims[1].size, token->dims[2].size, token->dims[3].size); + Tensor mlp_out = nullptr; for (int i = 0; i < mixtral_config.num_hidden_layers; i++) { @@ -91,6 +93,10 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, token = token_att_norm[0]; att_norm = token_att_norm[1]; } + + printf("token before MHA are %d %d %d %d\n", token->dims[0].size, token->dims[1].size, token->dims[2].size, token->dims[3].size); + + Tensor qkv_proj = ff.dense( att_norm, mixtral_config.hidden_size * @@ -166,6 +172,9 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, token = token_ff_norm[0]; Tensor ff_norm = token_ff_norm[1]; + printf("token before moe dims are %d %d %d %d\n", token->dims[0].size, token->dims[1].size, token->dims[2].size, token->dims[3].size); + + // MoE Tensor w1 = ff.dense( ff_norm, @@ -207,7 +216,10 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, REG_MODE_NONE, 0.0f, std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w2").c_str()); - } + + printf("mlp_out dims are %d %d %d %d\n", mlp_out->dims[0].size, mlp_out->dims[1].size, mlp_out->dims[2].size, mlp_out->dims[3].size); + + } // final normalization and linear Tensor final_rms_norm_output[2] = {nullptr, nullptr}; ff.residual_rms_norm(token, From bf65efe046b8080fe1ca73a708adaba35973a156 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 13:56:02 -0500 Subject: [PATCH 074/106] sync --- inference/models/mixtral.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index ccb66a44cc..c5c7eac643 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -60,7 +60,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, embed_init, "embed_tokens"); - printf("token before iteration dims are %d %d %d %d\n", token->dims[0].size, token->dims[1].size, token->dims[2].size, token->dims[3].size); + printf("token before iteration dims are %d %d %d %d\n", token->dims[0], token->dims[1], token->dims[2], token->dims[3]); Tensor mlp_out = nullptr; @@ -94,7 +94,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, att_norm = token_att_norm[1]; } - printf("token before MHA are %d %d %d %d\n", token->dims[0].size, token->dims[1].size, token->dims[2].size, token->dims[3].size); + printf("token before MHA are %d %d %d %d\n", token->dims[0], token->dims[1], token->dims[2], token->dims[3]); Tensor qkv_proj = ff.dense( @@ -172,7 +172,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, token = token_ff_norm[0]; Tensor ff_norm = token_ff_norm[1]; - printf("token before moe dims are %d %d %d %d\n", token->dims[0].size, token->dims[1].size, token->dims[2].size, token->dims[3].size); + printf("token before moe dims are %d %d %d %d\n", token->dims[0], token->dims[1], token->dims[2], token->dims[3]); // MoE @@ -217,7 +217,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 0.0f, std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w2").c_str()); - printf("mlp_out dims are %d %d %d %d\n", mlp_out->dims[0].size, mlp_out->dims[1].size, mlp_out->dims[2].size, mlp_out->dims[3].size); + printf("mlp_out dims are %d %d %d %d\n", mlp_out->dims[0], mlp_out->dims[1], mlp_out->dims[2], mlp_out->dims[3]); } // final normalization and linear From c989f5770e5cb364c00e5837a0c4ebbd70be4d9e Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 14:09:50 -0500 Subject: [PATCH 075/106] sync --- inference/models/mixtral.cc | 113 +++++++++++++++++++++++++++++++++--- 1 file changed, 106 insertions(+), 7 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index c5c7eac643..7891ff2d5d 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -15,6 +15,13 @@ #include "mixtral.h" +//#define MIXTRAL_DEBUG +#ifdef MIXTRAL_DEBUG +#define dbg_printf(...) printf(__VA_ARGS__) +#else +#define dbg_printf(...) +#endif + namespace FlexFlow { using namespace Legion; @@ -59,12 +66,13 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, NULL, embed_init, "embed_tokens"); - - printf("token before iteration dims are %d %d %d %d\n", token->dims[0], token->dims[1], token->dims[2], token->dims[3]); + // token has dimensions (hidden_size, 1, 128) Tensor mlp_out = nullptr; for (int i = 0; i < mixtral_config.num_hidden_layers; i++) { + dbg_printf("mixtral hidden layer %d\n", i); + // set transformer layer id ff.set_transformer_layer_id(i); @@ -93,8 +101,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, token = token_att_norm[0]; att_norm = token_att_norm[1]; } + // token has dimensions (hidden_size, 1, 128) - printf("token before MHA are %d %d %d %d\n", token->dims[0], token->dims[1], token->dims[2], token->dims[3]); Tensor qkv_proj = ff.dense( @@ -115,6 +123,50 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor mha; switch (mode) { + case BEAM_SEARCH_MODE: { + mha = ff.spec_inc_multiquery_self_attention( + qkv_proj, + mixtral_config.hidden_size, + mixtral_config.num_attention_heads, + mixtral_config.num_key_value_heads, + mixtral_config.hidden_size / mixtral_config.num_attention_heads, + mixtral_config.hidden_size / mixtral_config.num_attention_heads, + 0.0f, /*dropout*/ + false, /*add_zero_attn*/ + DT_NONE, /*data_type*/ + NULL, /*kernel_initializer*/ + mixtral_config.rotary_embedding_meta, + false, /*scaling query*/ + 1.0f, /*scaling factor*/ + true, /*qk_prod_scaling*/ + false, /*position_bias*/ + std::string("layers." + std::to_string(i) + ".self_attn") + .c_str() /*name*/ + ); + break; + } + case TREE_VERIFY_MODE: { + mha = ff.inc_multiquery_self_attention_verify( + qkv_proj, + mixtral_config.hidden_size, + mixtral_config.num_attention_heads, + mixtral_config.num_key_value_heads, + mixtral_config.hidden_size / mixtral_config.num_attention_heads, + mixtral_config.hidden_size / mixtral_config.num_attention_heads, + 0.0f, /*dropout*/ + false, /*add_zero_attn*/ + DT_NONE, /*data_type*/ + nullptr, /*kernel_initializer*/ + mixtral_config.rotary_embedding_meta, + false, /*scaling query*/ + 1.0f, /*scaling factor*/ + true, /*qk_prod_scaling*/ + false, /*position_bias*/ + std::string("layers." + std::to_string(i) + ".self_attn") + .c_str() /*name*/ + ); + break; + } case INC_DECODING_MODE: { mha = ff.inc_multiquery_self_attention( qkv_proj, @@ -169,13 +221,56 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, DT_NONE, std::string("layers." + std::to_string(i) + ".post_attention_layernorm") .c_str()); - token = token_ff_norm[0]; + token = token_ff_norm[0]; // token has dimensions (hidden_size, 1, 128) Tensor ff_norm = token_ff_norm[1]; - printf("token before moe dims are %d %d %d %d\n", token->dims[0], token->dims[1], token->dims[2], token->dims[3]); + // MoE + Tensor gate = ff.dense( + ff_norm, // (hidden_size, 1, 128) + mixtral_config.num_local_experts, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") + .c_str()); + + gate = ff.softmax( // This operation fails! + gate, // (num_experts, 1, 128) + 0, + DT_NONE, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_softmax") + .c_str()); + Tensor topk_out[2] = {nullptr, nullptr}; + ff.top_k( + gate, // (num_experts, 1, 128) + topk_out, + mixtral_config.num_experts_per_tok, + false, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") + .c_str()); + Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) (confirmed 3 dims) + Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) (confirmed 3 dims) + + Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; + ff.group_by( + ff_norm, + topk_indices, + grouped_tokens, + mixtral_config.num_local_experts, + 0.0f, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby") + .c_str()); + + // grouped_tokens[0] has dims (1024, 1, 0) + Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; + Tensor one_aggregate_inputs[1] = {nullptr}; - // MoE Tensor w1 = ff.dense( ff_norm, mixtral_config.intermediate_size, @@ -217,7 +312,11 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 0.0f, std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w2").c_str()); - printf("mlp_out dims are %d %d %d %d\n", mlp_out->dims[0], mlp_out->dims[1], mlp_out->dims[2], mlp_out->dims[3]); + // mlp_out has dimensions (hidden_size, 1, 128) + printf("mlp_out in layer %d dims are %d %d %d %d\n",i, mlp_out->dims[0], mlp_out->dims[1], mlp_out->dims[2], mlp_out->dims[3]); + assert(mlp_out->dims[0] == mixtral_config.hidden_size) && "mlp_out dims[0] != hidden_size"; + assert(mlp_out->dims[1] == 1) && "mlp_out dims[1] != 1"; + assert(mlp_out->dims[2] == 128) && "mlp_out dims[2] != 128"; } // final normalization and linear From f4c7d3a17e5709885499bdb3f35b56e591b9f197 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 14:10:54 -0500 Subject: [PATCH 076/106] sync --- inference/models/mixtral.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 7891ff2d5d..fb01390d57 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -314,9 +314,9 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, // mlp_out has dimensions (hidden_size, 1, 128) printf("mlp_out in layer %d dims are %d %d %d %d\n",i, mlp_out->dims[0], mlp_out->dims[1], mlp_out->dims[2], mlp_out->dims[3]); - assert(mlp_out->dims[0] == mixtral_config.hidden_size) && "mlp_out dims[0] != hidden_size"; - assert(mlp_out->dims[1] == 1) && "mlp_out dims[1] != 1"; - assert(mlp_out->dims[2] == 128) && "mlp_out dims[2] != 128"; + assert(mlp_out->dims[0] == mixtral_config.hidden_size && "mlp_out dims[0] != hidden_size"); + assert(mlp_out->dims[1] == 1 && "mlp_out dims[1] != 1"); + assert(mlp_out->dims[2] == 128 && "mlp_out dims[2] != 128"); } // final normalization and linear From 41cf63587cb4fc492675c16026e73ed2de0606ea Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 14:19:10 -0500 Subject: [PATCH 077/106] sync --- inference/models/mixtral.cc | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index fb01390d57..b31c12d844 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -257,15 +257,16 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) (confirmed 3 dims) Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) (confirmed 3 dims) - Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; - ff.group_by( - ff_norm, - topk_indices, - grouped_tokens, - mixtral_config.num_local_experts, - 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby") - .c_str()); + if (i < mixtral_config.num_hidden_layers - 1) { // TODO understand why that doesn't work for the last layer + Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; + ff.group_by( + ff_norm, + topk_indices, + grouped_tokens, + mixtral_config.num_local_experts, + 0.0f, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby").c_str()); + } // grouped_tokens[0] has dims (1024, 1, 0) Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; From 41213400c15502752b30ce3874ca28c867ef413e Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 14:22:11 -0500 Subject: [PATCH 078/106] sync --- inference/models/mixtral.cc | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index b31c12d844..0e439873e3 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -257,18 +257,19 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) (confirmed 3 dims) Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) (confirmed 3 dims) - if (i < mixtral_config.num_hidden_layers - 1) { // TODO understand why that doesn't work for the last layer - Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; - ff.group_by( - ff_norm, - topk_indices, - grouped_tokens, - mixtral_config.num_local_experts, - 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby").c_str()); - } +// TODO understand why graph.cc complains that last layer has multiple inputs +// Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; +// ff.group_by( +// ff_norm, +// topk_indices, +// grouped_tokens, +// mixtral_config.num_local_experts, +// 0.0f, +// std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby").c_str()); +// } // grouped_tokens[0] has dims (1024, 1, 0) + Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; Tensor one_aggregate_inputs[1] = {nullptr}; From 7f6a18b1c4f2138f1a3752f7bbf136f20dbc8c0e Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 14:28:06 -0500 Subject: [PATCH 079/106] sync --- inference/models/mixtral.cc | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 0e439873e3..399cbde683 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -239,7 +239,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") .c_str()); - gate = ff.softmax( // This operation fails! + gate = ff.softmax( gate, // (num_experts, 1, 128) 0, DT_NONE, @@ -247,15 +247,21 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, .c_str()); Tensor topk_out[2] = {nullptr, nullptr}; - ff.top_k( - gate, // (num_experts, 1, 128) - topk_out, - mixtral_config.num_experts_per_tok, - false, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") - .c_str()); - Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) (confirmed 3 dims) - Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) (confirmed 3 dims) + + /* TODO understand why I get the following error + [0 - 7238bc615000] 4.547830 {5}{runtime}: [error 545] LEGION ERROR: Error creating accessor for field 0 with + a type of size 4 bytes when the field was originally allocated with a size of 2 bytes in task TopK Forward + Task (UID 1698) (from file /home/FlexFlow/deps/legion/runtime/legion/runtime.cc:5451) + */ +// ff.top_k( +// gate, // (num_experts, 1, 128) +// topk_out, +// mixtral_config.num_experts_per_tok, +// false, +// std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") +// .c_str()); +// Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) (confirmed 3 dims) +// Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) (confirmed 3 dims) // TODO understand why graph.cc complains that last layer has multiple inputs // Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; From 349853857536f31da41b4c4bff3e55ab628370af Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 14:31:05 -0500 Subject: [PATCH 080/106] I'm able to output with only one expert by commenting out groupby and aggregate --- inference/models/mixtral.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 399cbde683..07fd25300d 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -276,6 +276,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, // grouped_tokens[0] has dims (1024, 1, 0) + Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; Tensor one_aggregate_inputs[1] = {nullptr}; From 344fe09d349f03c6af47900a172e7f41d4bcffc7 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 20:44:46 -0500 Subject: [PATCH 081/106] sync --- inference/models/mixtral.cc | 51 ++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 07fd25300d..d2b3521b69 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -280,6 +280,9 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; Tensor one_aggregate_inputs[1] = {nullptr}; + // TODO don't use only one expert + // for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; expert_idx++) { + for (int expert_idx = 1; expert_idx < 2; expert_idx++) { Tensor w1 = ff.dense( ff_norm, mixtral_config.intermediate_size, @@ -291,7 +294,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w1").c_str()); + std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_" + + std::to_string(expert_idx) + "_w1").c_str()); Tensor w3 = ff.dense( ff_norm, @@ -304,11 +308,12 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w3").c_str()); + std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_" + + std::to_string(expert_idx) + "_w3").c_str()); Tensor multi = ff.sigmoid_silu_multi(w1, w3); //DT_NONE,std::string("layers." + std::to_string(i) +".block_sparse_moe_experts." +std::to_string(expert_idx) + "ssm").c_str()); - mlp_out = ff.dense( + w2 = ff.dense( // output has dims (1024, 1, 0), 3 dims confirmed multi, mixtral_config.hidden_size, AC_MODE_NONE, @@ -319,7 +324,45 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, nullptr, REG_MODE_NONE, 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_1_w2").c_str()); + std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_" + + std::to_string(expert_idx) + "_w2").c_str()); + aggregate_inputs[4 + expert_idx] = w2; + } + +// Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) +// topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) +// Tensor dummy_gate = ff.dense( +// ff_norm, +// mixtral_config.num_local_experts, +// AC_MODE_NONE, +// false, +// DT_NONE, +// nullptr, +// nullptr, +// nullptr, +// REG_MODE_NONE, +// 0.0f, +// std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") +// .c_str()); +// dummy_gate = ff.softmax( +// gate, +// 0, +// DT_NONE, +// std::string("dummy_gate").c_str()); + +// aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) (3 dims confirmed) +// aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) (3 dims confirmed) +// aggregate_inputs[2] = topk_values; // TODO this is a tmp fix +// aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix + + mlp_out = aggregate_inputs[5]; // TODO don't use just one expert +// mlp_out = ff.aggregate(aggregate_inputs, +//// topk_values->dims[2], +// mixtral_config.num_local_experts, +// 0.0f, +// std::string("layers." + std::to_string(i) + +// ".block_sparse_moe_experts_aggregate") +// .c_str()); // mlp_out has dimensions (hidden_size, 1, 128) printf("mlp_out in layer %d dims are %d %d %d %d\n",i, mlp_out->dims[0], mlp_out->dims[1], mlp_out->dims[2], mlp_out->dims[3]); From b9883c13675c8a2f36b1111df01be164f44656c0 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 20:45:46 -0500 Subject: [PATCH 082/106] sync --- inference/models/mixtral.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index d2b3521b69..9b8c693fb6 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -313,7 +313,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor multi = ff.sigmoid_silu_multi(w1, w3); //DT_NONE,std::string("layers." + std::to_string(i) +".block_sparse_moe_experts." +std::to_string(expert_idx) + "ssm").c_str()); - w2 = ff.dense( // output has dims (1024, 1, 0), 3 dims confirmed + Tensor w2 = ff.dense( // output has dims (1024, 1, 0), 3 dims confirmed multi, mixtral_config.hidden_size, AC_MODE_NONE, @@ -326,7 +326,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, 0.0f, std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_" + std::to_string(expert_idx) + "_w2").c_str()); - aggregate_inputs[4 + expert_idx] = w2; + aggregate_inputs[4 + expert_idx] = w2; // (1024, 1, 0), 3 dims confirmed } // Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) From 65afc21a51c6acb30dc3c1a3b242f3ac4df9aefc Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 20:50:09 -0500 Subject: [PATCH 083/106] works without topk! --- inference/models/mixtral.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 9b8c693fb6..4040705416 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -249,7 +249,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor topk_out[2] = {nullptr, nullptr}; /* TODO understand why I get the following error - [0 - 7238bc615000] 4.547830 {5}{runtime}: [error 545] LEGION ERROR: Error creating accessor for field 0 with + [0 - 7238bc615000] 4.547830 {5}{runtime}: [error 545] LEGION ERROR: Error creating accessor for field 0 with a type of size 4 bytes when the field was originally allocated with a size of 2 bytes in task TopK Forward Task (UID 1698) (from file /home/FlexFlow/deps/legion/runtime/legion/runtime.cc:5451) */ From 4d6886c7ddb8d1e44b8d62a0f9437b9f66cc2d3a Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 21:09:24 -0500 Subject: [PATCH 084/106] outputs tokens with softmax assertion not commented out --- inference/models/mixtral.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 4040705416..4571c858ec 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -248,6 +248,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor topk_out[2] = {nullptr, nullptr}; + /* TODO understand why I get the following error [0 - 7238bc615000] 4.547830 {5}{runtime}: [error 545] LEGION ERROR: Error creating accessor for field 0 with a type of size 4 bytes when the field was originally allocated with a size of 2 bytes in task TopK Forward From e757567b55e2a72309fc2d73122ca671e9175cc1 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 21:11:59 -0500 Subject: [PATCH 085/106] sync --- inference/models/mixtral.cc | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 4571c858ec..8c43afb4c6 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -246,7 +246,6 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::string("layers." + std::to_string(i) + ".block_sparse_moe_softmax") .c_str()); - Tensor topk_out[2] = {nullptr, nullptr}; /* TODO understand why I get the following error @@ -254,15 +253,17 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, a type of size 4 bytes when the field was originally allocated with a size of 2 bytes in task TopK Forward Task (UID 1698) (from file /home/FlexFlow/deps/legion/runtime/legion/runtime.cc:5451) */ -// ff.top_k( -// gate, // (num_experts, 1, 128) -// topk_out, -// mixtral_config.num_experts_per_tok, -// false, -// std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk") -// .c_str()); -// Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) (confirmed 3 dims) -// Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) (confirmed 3 dims) + Tensor topk_out[2] = {nullptr, nullptr}; + + ff.top_k( + gate, // (num_experts, 1, 128) + topk_out, + mixtral_config.num_experts_per_tok, + false, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk").c_str() + ); + Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) (confirmed 3 dims) + Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) (confirmed 3 dims) // TODO understand why graph.cc complains that last layer has multiple inputs // Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; From d45edbcc488ea6eb0fdc8c15e974a4ab0fb648a5 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 21:48:12 -0500 Subject: [PATCH 086/106] sync --- inference/models/mixtral.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 8c43afb4c6..0be0eb341c 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -254,7 +254,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Task (UID 1698) (from file /home/FlexFlow/deps/legion/runtime/legion/runtime.cc:5451) */ Tensor topk_out[2] = {nullptr, nullptr}; - + printf("gate data_type %d\n", gate->data_type); ff.top_k( gate, // (num_experts, 1, 128) topk_out, From 8c422aaa0baeaa31c0f799c3cd8e607c863428d2 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:01:58 -0500 Subject: [PATCH 087/106] use full precisoin --- inference/spec_infer/spec_infer.cc | 2 +- src/ops/aggregate.cc | 69 ++++++++++++++++-------------- src/ops/group_by.cc | 9 ++-- 3 files changed, 43 insertions(+), 37 deletions(-) diff --git a/inference/spec_infer/spec_infer.cc b/inference/spec_infer/spec_infer.cc index dce77d1e30..7041906a3d 100644 --- a/inference/spec_infer/spec_infer.cc +++ b/inference/spec_infer/spec_infer.cc @@ -290,7 +290,7 @@ void FlexFlow::top_level_task(Task const *task, FFConfig ffconfig; FilePaths file_paths; ModelMeta model_metadata; - bool use_full_precision = false; + bool use_full_precision = true; bool verbose = false; int max_requests_per_batch = 16; int max_tokens_per_batch = 256; diff --git a/src/ops/aggregate.cc b/src/ops/aggregate.cc index 3440a81533..b2fc61b5cd 100644 --- a/src/ops/aggregate.cc +++ b/src/ops/aggregate.cc @@ -37,6 +37,9 @@ using Legion::TaskArgument; using Legion::TaskLauncher; using PCG::Node; +// Number of inputs that are not expert predictions +#define FIXED_ARG_CNT 4 + // This runs when mixtral.cc is run Tensor FFModel::aggregate( Tensor const *inputs, /* gate_preds, gate_assign, gate assign TopK, @@ -46,9 +49,9 @@ Tensor FFModel::aggregate( char const *name) { Layer *li = new Layer(this, OP_AGGREGATE, - DT_FLOAT, + inputs[FIXED_ARG_CNT]->data_type, name, - n + 4 /*num inputs*/, + n + FIXED_ARG_CNT /*num inputs*/, 0 /*weights*/, 1 /*outputs*/, inputs); @@ -56,15 +59,15 @@ Tensor FFModel::aggregate( printf("In FFModel::aggregate, inputs[0]->num_dims = %d\n", inputs[0]->num_dims); - int num_dim = inputs[4]->num_dims; + int num_dim = inputs[FIXED_ARG_CNT]->num_dims; // Set output shape int dims[MAX_TENSOR_DIM]; for (int i = 0; i < num_dim - 1; i++) { - dims[i] = inputs[4]->dims[i]; + dims[i] = inputs[FIXED_ARG_CNT]->dims[i]; } dims[num_dim - 1] = inputs[0]->dims[num_dim - 1]; li->outputs[0] = create_tensor_legion_ordering( - num_dim, dims, DT_FLOAT, li, 0, true /*create_grad*/); + num_dim, dims, inputs[FIXED_ARG_CNT]->data_type, li, 0, true /*create_grad*/); } li->add_int_property("n", n); li->add_float_property("lambda_bal", lambda_bal); @@ -114,7 +117,7 @@ Aggregate::Aggregate(FFModel &model, OP_AGGREGATE, DT_FLOAT, name, - _n + 4 /*numInputs*/, + _n + FIXED_ARG_CNT /*numInputs*/, 0 /*numWeights*/, 1 /*numOutputs*/, _inputs), @@ -139,7 +142,7 @@ Aggregate::Aggregate(FFModel &model, // assert(inputs[0]->dims[1].size <= AGGREGATE_MAX_BATCH_SIZE && // "Increase AGGREGATE_MAX_BATCH_SIZE in #define"); // -// assert(n + 4 == numInputs); +// assert(n + FIXED_ARG_CNT == numInputs); // assert(n > 0); // //printf("In Aggregate::Aggregate, inputs[0]->num_dims = %d\n", inputs[0]->num_dims); // //printf("In Aggregate::Aggregate, inputs[0] dims are %d %d %d %d\n", inputs[0]->dims[0].size, inputs[0]->dims[1].size, inputs[0]->dims[2].size, inputs[0]->dims[3].size); @@ -157,16 +160,16 @@ Aggregate::Aggregate(FFModel &model, // assert(inputs[3]->dims[0].size == n); // expert inputs - int num_dim = inputs[4]->num_dims; // 3 - int out_dim = inputs[4]->dims[0].size; + int num_dim = inputs[FIXED_ARG_CNT]->num_dims; // 3 + int out_dim = inputs[FIXED_ARG_CNT]->dims[0].size; // for (int i = 1; i < n; i++) { -// assert(inputs[i + 4]->num_dims == num_dim); -// assert(inputs[i + 4]->dims[0].size == out_dim); +// assert(inputs[i + FIXED_ARG_CNT]->num_dims == num_dim); +// assert(inputs[i + FIXED_ARG_CNT]->dims[0].size == out_dim); // } // Set output shape ParallelDim dims[MAX_TENSOR_DIM]; for (int i = 0; i < num_dim - 1; i++) { - dims[i] = inputs[4]->dims[i]; + dims[i] = inputs[FIXED_ARG_CNT]->dims[i]; } // TODO replace with inputs[0]->dims[num_dim - 2] @@ -217,7 +220,7 @@ Node Aggregate::deserialize(FFModel &ff, char name[MAX_OPNAME] = {0}; dez.deserialize(name_len); dez.deserialize(name, name_len); - assert(num_inputs == n + 4); + assert(num_inputs == n + FIXED_ARG_CNT); AggregateParams params; params.n = n; params.lambda_bal = lambda_bal; @@ -315,11 +318,11 @@ void Aggregate::forward(FFModel const &ff) { launcher.add_field(1, FID_DATA); // exp_preds for (int i = 0; i < n; i++) { - launcher.add_region_requirement(RegionRequirement(inputs[i + 4]->part, + launcher.add_region_requirement(RegionRequirement(inputs[i + FIXED_ARG_CNT]->part, 0 /*projection id*/, READ_WRITE, EXCLUSIVE, - inputs[i + 4]->region)); + inputs[i + FIXED_ARG_CNT]->region)); launcher.add_field(i + 2, FID_DATA); } // output @@ -371,11 +374,11 @@ FutureMap Aggregate::inference(FFModel const &ff, // exp_preds for (int i = 0; i < n; i++) { launcher.add_region_requirement( - RegionRequirement(batch_inputs[i + 4]->part, + RegionRequirement(batch_inputs[i + FIXED_ARG_CNT]->part, 0 /*projection id*/, READ_WRITE, EXCLUSIVE, - batch_inputs[i + 4]->region)); + batch_inputs[i + FIXED_ARG_CNT]->region)); launcher.add_field(i + 2, FID_DATA); } // output @@ -493,22 +496,22 @@ void Aggregate::backward(FFModel const &ff) { launcher.add_field(3, FID_DATA); // exp_preds for (int i = 0; i < n; i++) { - launcher.add_region_requirement(RegionRequirement(inputs[i + 4]->part, + launcher.add_region_requirement(RegionRequirement(inputs[i + FIXED_ARG_CNT]->part, 0 /*projection id*/, READ_WRITE, EXCLUSIVE, - inputs[i + 4]->region)); - launcher.add_field(i + 4, FID_DATA); + inputs[i + FIXED_ARG_CNT]->region)); + launcher.add_field(i + FIXED_ARG_CNT, FID_DATA); } // exp_preds gradients for (int i = 0; i < n; i++) { launcher.add_region_requirement( - RegionRequirement(inputs[i + 4]->part_grad, + RegionRequirement(inputs[i + FIXED_ARG_CNT]->part_grad, 0 /*projection id*/, READ_WRITE, EXCLUSIVE, - inputs[i + 4]->region_grad)); - launcher.add_field(i + n + 4, FID_DATA); + inputs[i + FIXED_ARG_CNT]->region_grad)); + launcher.add_field(i + n + FIXED_ARG_CNT, FID_DATA); } // output @@ -517,7 +520,7 @@ void Aggregate::backward(FFModel const &ff) { READ_WRITE, EXCLUSIVE, outputs[0]->region_grad)); - launcher.add_field(2 * n + 4, FID_DATA); + launcher.add_field(2 * n + FIXED_ARG_CNT, FID_DATA); runtime->execute_index_space(ctx, launcher); } @@ -538,7 +541,7 @@ void Aggregate::backward_task(Task const *task, AccessorRO const acc_gate_assign(regions[1], FID_DATA); AccessorRO const acc_true_gate_assign(regions[2], FID_DATA); AccessorWO const full_acc_gate_grad(regions[3], FID_DATA); - AccessorRO const acc_output_grad(regions[2 * n + 4], FID_DATA); + AccessorRO const acc_output_grad(regions[2 * n + FIXED_ARG_CNT], FID_DATA); Rect<3> rect_gate_pred = runtime->get_index_space_domain( ctx, task->regions[0].region.get_index_space()); @@ -549,7 +552,7 @@ void Aggregate::backward_task(Task const *task, Rect<3> rect_full_gate_grad = runtime->get_index_space_domain( ctx, task->regions[3].region.get_index_space()); Rect<3> rect_out_grad = runtime->get_index_space_domain( - ctx, task->regions[2 * n + 4].region.get_index_space()); + ctx, task->regions[2 * n + FIXED_ARG_CNT].region.get_index_space()); coord_t batch_size = rect_gate_pred.hi[1] - rect_gate_pred.lo[1] + 1; assert(batch_size == rect_gate_assign.hi[1] - rect_gate_assign.lo[1] + 1); @@ -566,17 +569,17 @@ void Aggregate::backward_task(Task const *task, float *exp_preds[n]; // get first exp_pred and row Domain exp_domain = runtime->get_index_space_domain( - ctx, task->regions[4].region.get_index_space()); + ctx, task->regions[FIXED_ARG_CNT].region.get_index_space()); exp_preds[0] = helperGetTensorPointerRW( - regions[4], task->regions[4], FID_DATA, ctx, runtime); + regions[FIXED_ARG_CNT], task->regions[FIXED_ARG_CNT], FID_DATA, ctx, runtime); coord_t rows = exp_domain.hi()[1] - exp_domain.lo()[1] + 1; assert(out_dim == exp_domain.hi()[0] - exp_domain.lo()[0] + 1); for (int i = 1; i < n; i++) { exp_domain = runtime->get_index_space_domain( - ctx, task->regions[i + 4].region.get_index_space()); + ctx, task->regions[i + FIXED_ARG_CNT].region.get_index_space()); exp_preds[i] = helperGetTensorPointerRW( - regions[i + 4], task->regions[i + 4], FID_DATA, ctx, runtime); + regions[i + FIXED_ARG_CNT], task->regions[i + FIXED_ARG_CNT], FID_DATA, ctx, runtime); assert(rows == exp_domain.hi()[1] - exp_domain.lo()[1] + 1); assert(out_dim == exp_domain.hi()[0] - exp_domain.lo()[0] + 1); } @@ -585,9 +588,9 @@ void Aggregate::backward_task(Task const *task, float *exp_grads[n]; for (int i = 0; i < n; i++) { exp_domain = runtime->get_index_space_domain( - ctx, task->regions[n + i + 4].region.get_index_space()); + ctx, task->regions[n + i + FIXED_ARG_CNT].region.get_index_space()); exp_grads[i] = helperGetTensorPointerRW( - regions[n + i + 4], task->regions[n + i + 4], FID_DATA, ctx, runtime); + regions[n + i + FIXED_ARG_CNT], task->regions[n + i + FIXED_ARG_CNT], FID_DATA, ctx, runtime); assert(rows == exp_domain.hi()[1] - exp_domain.lo()[1] + 1); assert(out_dim == exp_domain.hi()[0] - exp_domain.lo()[0] + 1); } @@ -624,7 +627,7 @@ bool Aggregate::measure_operator_cost(Simulator *sim, sub_output; for (int i = 0; i < numInputs; ++i) { - if (!inputs[i + 4]->get_sub_tensor(mv, sub_inputs[i])) { + if (!inputs[i + FIXED_ARG_CNT]->get_sub_tensor(mv, sub_inputs[i])) { return false; } } diff --git a/src/ops/group_by.cc b/src/ops/group_by.cc index a1788ed033..f941b523a7 100644 --- a/src/ops/group_by.cc +++ b/src/ops/group_by.cc @@ -47,9 +47,12 @@ void FFModel::group_by(const Tensor input, int n, float alpha, char const *name) { + + DataType data_type = input->data_type; + Layer *li = new Layer(this, OP_GROUP_BY, - DT_FLOAT, + data_type, name, 2 /*inputs*/, 0 /*weights*/, @@ -149,7 +152,7 @@ Group_by::Group_by(FFModel &model, for (int i = 0; i < n; i++) { outputs[i] = model.create_parallel_tensor_legion_ordering( - num_dims, dims, DT_FLOAT, this, i /*owner_idx*/); + num_dims, dims, data_type, this, i /*owner_idx*/); assert(outputs[i] != nullptr); } @@ -335,7 +338,7 @@ void Group_by::forward_task(Task const *task, // get input and assign regions. Each tensor has three dimensions: // (datapoint_dim, batch_size, replica_dim) GenericTensorAccessorR input = helperGetGenericTensorAccessorRO( - DT_FLOAT, regions[0], task->regions[0], FID_DATA, ctx, runtime); + data_type, regions[0], task->regions[0], FID_DATA, ctx, runtime); GenericTensorAccessorR assign = helperGetGenericTensorAccessorRO( DT_INT32, regions[1], task->regions[1], FID_DATA, ctx, runtime); Domain input_domain = runtime->get_index_space_domain( From c0e48244bfda03414344911abe1e3d584b2fff93 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:03:07 -0500 Subject: [PATCH 088/106] sync --- src/ops/group_by.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ops/group_by.cc b/src/ops/group_by.cc index f941b523a7..65ca42625e 100644 --- a/src/ops/group_by.cc +++ b/src/ops/group_by.cc @@ -338,7 +338,7 @@ void Group_by::forward_task(Task const *task, // get input and assign regions. Each tensor has three dimensions: // (datapoint_dim, batch_size, replica_dim) GenericTensorAccessorR input = helperGetGenericTensorAccessorRO( - data_type, regions[0], task->regions[0], FID_DATA, ctx, runtime); + DT_FLOAT, regions[0], task->regions[0], FID_DATA, ctx, runtime); GenericTensorAccessorR assign = helperGetGenericTensorAccessorRO( DT_INT32, regions[1], task->regions[1], FID_DATA, ctx, runtime); Domain input_domain = runtime->get_index_space_domain( From 1c13c1e0ad7fb7b49545e12858b8517fe5e2e532 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:08:12 -0500 Subject: [PATCH 089/106] sync --- rename.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rename.sh b/rename.sh index 63e7d5f5a3..73a01c7dd7 100755 --- a/rename.sh +++ b/rename.sh @@ -1,7 +1,7 @@ #!/bin/bash # Target directory -TARGET_DIR="/root/.cache/flexflow/weights/m4-ai/tinymistral-6x248m/half-precision" +TARGET_DIR="/root/.cache/flexflow/weights/m4-ai/tinymistral-6x248m/full-precision" # Loop through files containing "layer_" for file in "$TARGET_DIR"/*layer_*; do From d31cbea49ec2a6c3c00f017397a9ac8c0c8fca69 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:10:14 -0500 Subject: [PATCH 090/106] sync --- rename.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rename.sh b/rename.sh index 73a01c7dd7..6d18c1d7b1 100755 --- a/rename.sh +++ b/rename.sh @@ -59,4 +59,10 @@ for file in "$TARGET_DIR"/*attn_*; do if [[ -f "$file" ]]; then mv "$file" "${file//attn_/attn.}" fi +done + +for file in "$TARGET_DIR"/*embed_token_weight*; do + if [[ -f "$file" ]]; then + mv "$file" "${file//embed_token_weight/embed_token.weight}" + fi done \ No newline at end of file From 8c525c489503eeb8d0f91c37d19250681ce1becb Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:11:53 -0500 Subject: [PATCH 091/106] sync --- rename.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rename.sh b/rename.sh index 6d18c1d7b1..fecb1d7384 100755 --- a/rename.sh +++ b/rename.sh @@ -61,8 +61,8 @@ for file in "$TARGET_DIR"/*attn_*; do fi done -for file in "$TARGET_DIR"/*embed_token_weight*; do +for file in "$TARGET_DIR"/*embed_tokens_weight*; do if [[ -f "$file" ]]; then - mv "$file" "${file//embed_token_weight/embed_token.weight}" + mv "$file" "${file//embed_tokens_weight/embed_tokens.weight}" fi done \ No newline at end of file From 8b10df9ae5dbc87fa4e18a67289f8a523a7595e6 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:13:11 -0500 Subject: [PATCH 092/106] sync --- rename.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rename.sh b/rename.sh index fecb1d7384..0f75d34f25 100755 --- a/rename.sh +++ b/rename.sh @@ -65,4 +65,10 @@ for file in "$TARGET_DIR"/*embed_tokens_weight*; do if [[ -f "$file" ]]; then mv "$file" "${file//embed_tokens_weight/embed_tokens.weight}" fi +done + +for file in "$TARGET_DIR"/*layernorm_weight*; do + if [[ -f "$file" ]]; then + mv "$file" "${file//layernorm_weight/layernorm.weight}" + fi done \ No newline at end of file From f8ea430e10528cad26f5d610d39e3d84b0d4cd9c Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:14:37 -0500 Subject: [PATCH 093/106] sync --- rename.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rename.sh b/rename.sh index 0f75d34f25..29684cc8c9 100755 --- a/rename.sh +++ b/rename.sh @@ -71,4 +71,11 @@ for file in "$TARGET_DIR"/*layernorm_weight*; do if [[ -f "$file" ]]; then mv "$file" "${file//layernorm_weight/layernorm.weight}" fi +done + + +for file in "$TARGET_DIR"/*_weight*; do + if [[ -f "$file" ]]; then + mv "$file" "${file//_weight/.weight}" + fi done \ No newline at end of file From 54e6adc180261ee9712c36fdc5f5c58ab6373a33 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:19:11 -0500 Subject: [PATCH 094/106] sync --- inference/incr_decoding/incr_decoding.cc | 2 +- inference/models/mixtral.cc | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/inference/incr_decoding/incr_decoding.cc b/inference/incr_decoding/incr_decoding.cc index c19ee02580..d01cd98574 100644 --- a/inference/incr_decoding/incr_decoding.cc +++ b/inference/incr_decoding/incr_decoding.cc @@ -102,7 +102,7 @@ void parse_input_args(char **argv, max_tokens_per_batch = std::stoi(argv[++i]); continue; } - if (!strcmp(argv[i], "--max-sequence-length")) { + if (!strcmp(argv[i], "--max-sequence-length")) { // total including prompt max_sequence_length = std::stoi(argv[++i]); continue; } diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 0be0eb341c..17a1d86dfa 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -370,7 +370,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, printf("mlp_out in layer %d dims are %d %d %d %d\n",i, mlp_out->dims[0], mlp_out->dims[1], mlp_out->dims[2], mlp_out->dims[3]); assert(mlp_out->dims[0] == mixtral_config.hidden_size && "mlp_out dims[0] != hidden_size"); assert(mlp_out->dims[1] == 1 && "mlp_out dims[1] != 1"); - assert(mlp_out->dims[2] == 128 && "mlp_out dims[2] != 128"); +// assert(mlp_out->dims[2] == 128 && "mlp_out dims[2] != 128"); // TODO update with new seq len + printf("seq length is now %d\n", mlp_out->dims[2]); } // final normalization and linear From e5075fa13b6810ed234365aa0fee69150c42b9c0 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:28:08 -0500 Subject: [PATCH 095/106] sync --- inference/models/mixtral.cc | 50 ++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 17a1d86dfa..efcdb60df5 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -331,31 +331,31 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[4 + expert_idx] = w2; // (1024, 1, 0), 3 dims confirmed } -// Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) -// topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) -// Tensor dummy_gate = ff.dense( -// ff_norm, -// mixtral_config.num_local_experts, -// AC_MODE_NONE, -// false, -// DT_NONE, -// nullptr, -// nullptr, -// nullptr, -// REG_MODE_NONE, -// 0.0f, -// std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") -// .c_str()); -// dummy_gate = ff.softmax( -// gate, -// 0, -// DT_NONE, -// std::string("dummy_gate").c_str()); - -// aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) (3 dims confirmed) -// aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) (3 dims confirmed) -// aggregate_inputs[2] = topk_values; // TODO this is a tmp fix -// aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix + Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) + topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) + Tensor dummy_gate = ff.dense( + ff_norm, + mixtral_config.num_local_experts, + AC_MODE_NONE, + false, + DT_NONE, + nullptr, + nullptr, + nullptr, + REG_MODE_NONE, + 0.0f, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") + .c_str()); + dummy_gate = ff.softmax( + gate, + 0, + DT_NONE, + std::string("dummy_gate").c_str()); + + aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) (3 dims confirmed) + aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) (3 dims confirmed) + aggregate_inputs[2] = topk_values; // TODO this is a tmp fix + aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix mlp_out = aggregate_inputs[5]; // TODO don't use just one expert // mlp_out = ff.aggregate(aggregate_inputs, From fe8e7fdc2e14c27d80ff1bbcd40efac5a4446dee Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:31:11 -0500 Subject: [PATCH 096/106] sync --- inference/models/mixtral.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index efcdb60df5..2da610fe69 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -332,7 +332,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, } Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) - topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) +// topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) Tensor dummy_gate = ff.dense( ff_norm, mixtral_config.num_local_experts, From dbe3be6dc78d50c1ab0d828472879ce9261c75af Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:35:31 -0500 Subject: [PATCH 097/106] sync --- inference/models/mixtral.cc | 48 ++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 2da610fe69..cac6a5e96c 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -332,30 +332,30 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, } Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) -// topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) - Tensor dummy_gate = ff.dense( - ff_norm, - mixtral_config.num_local_experts, - AC_MODE_NONE, - false, - DT_NONE, - nullptr, - nullptr, - nullptr, - REG_MODE_NONE, - 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") - .c_str()); - dummy_gate = ff.softmax( - gate, - 0, - DT_NONE, - std::string("dummy_gate").c_str()); - - aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) (3 dims confirmed) - aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) (3 dims confirmed) - aggregate_inputs[2] = topk_values; // TODO this is a tmp fix - aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix +// topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) // TODO causes an error +// Tensor dummy_gate = ff.dense( +// ff_norm, +// mixtral_config.num_local_experts, +// AC_MODE_NONE, +// false, +// DT_NONE, +// nullptr, +// nullptr, +// nullptr, +// REG_MODE_NONE, +// 0.0f, +// std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") +// .c_str()); +// dummy_gate = ff.softmax( +// gate, +// 0, +// DT_NONE, +// std::string("dummy_gate").c_str()); +// +// aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) (3 dims confirmed) +// aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) (3 dims confirmed) +// aggregate_inputs[2] = topk_values; // TODO this is a tmp fix +// aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix mlp_out = aggregate_inputs[5]; // TODO don't use just one expert // mlp_out = ff.aggregate(aggregate_inputs, From 0d2fa44e9ac69a1e82f9d7314de088673fb39191 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:37:28 -0500 Subject: [PATCH 098/106] sync --- inference/models/mixtral.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index cac6a5e96c..816e4529ff 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -331,9 +331,9 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[4 + expert_idx] = w2; // (1024, 1, 0), 3 dims confirmed } - Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) +// Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) // TODO latest // topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) // TODO causes an error -// Tensor dummy_gate = ff.dense( +// Tensor dummy_gate = ff.dense( // TODO try uncommenting the whole block // ff_norm, // mixtral_config.num_local_experts, // AC_MODE_NONE, From a0ad4c084973cecc625416af883edb6736716a33 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:39:10 -0500 Subject: [PATCH 099/106] sync --- inference/models/mixtral.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 816e4529ff..221e75b4c9 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -331,7 +331,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[4 + expert_idx] = w2; // (1024, 1, 0), 3 dims confirmed } -// Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) // TODO latest + Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) // TODO latest // topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) // TODO causes an error // Tensor dummy_gate = ff.dense( // TODO try uncommenting the whole block // ff_norm, From 5473f2c994be34d5b8a209b3643de8b43e148562 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Sun, 8 Dec 2024 22:40:46 -0500 Subject: [PATCH 100/106] sync --- inference/models/mixtral.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 221e75b4c9..ca0a618882 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -331,7 +331,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[4 + expert_idx] = w2; // (1024, 1, 0), 3 dims confirmed } - Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) // TODO latest +// Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) // // topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) // TODO causes an error // Tensor dummy_gate = ff.dense( // TODO try uncommenting the whole block // ff_norm, From 927356d256140803d1fda4644e5018c018333cc2 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Mon, 9 Dec 2024 10:07:08 -0500 Subject: [PATCH 101/106] sync --- inference/models/mixtral.cc | 35 +++++++++++++++--------------- inference/spec_infer/spec_infer.cc | 2 +- src/ops/aggregate.cc | 2 ++ 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index ca0a618882..2500c67c0e 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -88,6 +88,10 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::string("layers." + std::to_string(i) + ".input_layernorm") .c_str()); } else { + printf("before rms norm in layer %d token has %d dims\n",i, token->num_dims); + printf("before rms norm in layer %d mlp_out has %d dims\n",i, token->num_dims); + printf("before rms norm in layer %d token dims are %d %d %d %d\n",i, token->dims[0], token->dims[1], token->dims[2], token->dims[3]); + printf("before rms norm in layer %d, mlp_out dims are %d %d %d %d\n",i, mlp_out->dims[0], mlp_out->dims[1], mlp_out->dims[2], mlp_out->dims[3]); ff.residual_rms_norm( token, mlp_out, @@ -239,7 +243,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") .c_str()); - gate = ff.softmax( + gate = ff.softmax( // TODO This sfotmax is wrong! not taking across last dim, which is not supported by ff! gate, // (num_experts, 1, 128) 0, DT_NONE, @@ -247,12 +251,6 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, .c_str()); - - /* TODO understand why I get the following error - [0 - 7238bc615000] 4.547830 {5}{runtime}: [error 545] LEGION ERROR: Error creating accessor for field 0 with - a type of size 4 bytes when the field was originally allocated with a size of 2 bytes in task TopK Forward - Task (UID 1698) (from file /home/FlexFlow/deps/legion/runtime/legion/runtime.cc:5451) - */ Tensor topk_out[2] = {nullptr, nullptr}; printf("gate data_type %d\n", gate->data_type); ff.top_k( @@ -331,8 +329,11 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[4 + expert_idx] = w2; // (1024, 1, 0), 3 dims confirmed } -// Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) // -// topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) // TODO causes an error + // TODO those two lines are techincally nice-to-haves!! skip for now, but it fails if we uncomment +// Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) +// topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) + + // Tensor dummy_gate = ff.dense( // TODO try uncommenting the whole block // ff_norm, // mixtral_config.num_local_experts, @@ -352,19 +353,17 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, // DT_NONE, // std::string("dummy_gate").c_str()); // -// aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) (3 dims confirmed) -// aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) (3 dims confirmed) -// aggregate_inputs[2] = topk_values; // TODO this is a tmp fix -// aggregate_inputs[3] = dummy_gate; // TODO this is a tmp fix + aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) (3 dims confirmed) + aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) (3 dims confirmed) + aggregate_inputs[2] = topk_values; // TODO this is a tmp fix + aggregate_inputs[3] = gate; // TODO this is a tmp fix mlp_out = aggregate_inputs[5]; // TODO don't use just one expert // mlp_out = ff.aggregate(aggregate_inputs, //// topk_values->dims[2], -// mixtral_config.num_local_experts, +// mixtral_config.num_local_experts, // TODO don't use just one expert // 0.0f, -// std::string("layers." + std::to_string(i) + -// ".block_sparse_moe_experts_aggregate") -// .c_str()); +// std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_aggregate").c_str()); // mlp_out has dimensions (hidden_size, 1, 128) printf("mlp_out in layer %d dims are %d %d %d %d\n",i, mlp_out->dims[0], mlp_out->dims[1], mlp_out->dims[2], mlp_out->dims[3]); @@ -404,7 +403,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor softmax = ff.softmax(dense, -1); output = ff.sampling(softmax, generation_config.topp); } else { - Tensor softmax = ff.softmax(dense, -1); // TODO added that to copy llama + Tensor softmax = ff.softmax(dense, -1); // TODO added that to copy llama, see if needed in HF transformers impl. output = ff.argmax(softmax, /*beam_Search*/ false); } diff --git a/inference/spec_infer/spec_infer.cc b/inference/spec_infer/spec_infer.cc index 7041906a3d..dce77d1e30 100644 --- a/inference/spec_infer/spec_infer.cc +++ b/inference/spec_infer/spec_infer.cc @@ -290,7 +290,7 @@ void FlexFlow::top_level_task(Task const *task, FFConfig ffconfig; FilePaths file_paths; ModelMeta model_metadata; - bool use_full_precision = true; + bool use_full_precision = false; bool verbose = false; int max_requests_per_batch = 16; int max_tokens_per_batch = 256; diff --git a/src/ops/aggregate.cc b/src/ops/aggregate.cc index b2fc61b5cd..d9907473a9 100644 --- a/src/ops/aggregate.cc +++ b/src/ops/aggregate.cc @@ -136,6 +136,8 @@ Aggregate::Aggregate(FFModel &model, printf("_inputs[0]->dims[2].parallel_idx = %d\n", _inputs[0]->dims[2].parallel_idx); printf("_inputs[0]->dims[2].is_replica_dim = %d\n", _inputs[0]->dims[2].is_replica_dim); + + // TODO uncomment all those assertions // assert(n <= AGGREGATE_MAX_N && "Increase AGGREGATE_MAX_N in #define"); // assert(inputs[0]->dims[0].size <= AGGREGATE_MAX_K && // "Increase AGGREGATE_MAX_K in #define"); From f7c6360511a9f1d5757a03b9fa236a9531c9c6a3 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Mon, 9 Dec 2024 10:14:47 -0500 Subject: [PATCH 102/106] sync --- inference/models/mixtral.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 2500c67c0e..fa23d73355 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -281,8 +281,8 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor one_aggregate_inputs[1] = {nullptr}; // TODO don't use only one expert - // for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; expert_idx++) { - for (int expert_idx = 1; expert_idx < 2; expert_idx++) { + for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; expert_idx++) { +// for (int expert_idx = 1; expert_idx < 2; expert_idx++) { Tensor w1 = ff.dense( ff_norm, mixtral_config.intermediate_size, @@ -329,7 +329,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, aggregate_inputs[4 + expert_idx] = w2; // (1024, 1, 0), 3 dims confirmed } - // TODO those two lines are techincally nice-to-haves!! skip for now, but it fails if we uncomment + // TODO those two lines are techincally nice-to-haves!! skip for now, but it fails if we uncomment // Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) // topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) @@ -353,10 +353,10 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, // DT_NONE, // std::string("dummy_gate").c_str()); // - aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) (3 dims confirmed) + aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) (3 dims confirmed) aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) (3 dims confirmed) aggregate_inputs[2] = topk_values; // TODO this is a tmp fix - aggregate_inputs[3] = gate; // TODO this is a tmp fix + aggregate_inputs[3] = gate; // TODO this is a tmp fix TODO decide vs dummygate mlp_out = aggregate_inputs[5]; // TODO don't use just one expert // mlp_out = ff.aggregate(aggregate_inputs, From 88a71637c206c93ed72256f780160e1b7350225a Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Mon, 9 Dec 2024 10:16:37 -0500 Subject: [PATCH 103/106] just run groupby --- inference/models/mixtral.cc | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index fa23d73355..a8708e13fe 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -264,15 +264,14 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) (confirmed 3 dims) // TODO understand why graph.cc complains that last layer has multiple inputs -// Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; -// ff.group_by( -// ff_norm, -// topk_indices, -// grouped_tokens, -// mixtral_config.num_local_experts, -// 0.0f, -// std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby").c_str()); -// } + Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; + ff.group_by( + ff_norm, + topk_indices, + grouped_tokens, + mixtral_config.num_local_experts, + 0.0f, + std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby").c_str()); // grouped_tokens[0] has dims (1024, 1, 0) From 89f5a855121629802e2fc8008f264bc8cdb083b5 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Mon, 9 Dec 2024 10:36:49 -0500 Subject: [PATCH 104/106] sync --- inference/models/mixtral.cc | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index a8708e13fe..7fb0f6049f 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -258,24 +258,23 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, topk_out, mixtral_config.num_experts_per_tok, false, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk").c_str() + std::string("layers." + std::to_string(i) + ".block_sparse_moe_topk").c_str() // No corresponding weights ); Tensor topk_values = topk_out[0]; // (experts_per_tok, 1, 128) (confirmed 3 dims) Tensor topk_indices = topk_out[1]; // (experts_per_tok, 1, 128) (confirmed 3 dims) -// TODO understand why graph.cc complains that last layer has multiple inputs - Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; - ff.group_by( - ff_norm, - topk_indices, - grouped_tokens, - mixtral_config.num_local_experts, - 0.0f, - std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby").c_str()); +// TODO understand why this causes graph.cc complains that last layer has multiple inputs +// Tensor grouped_tokens[mixtral_config.num_local_experts] = {nullptr}; +// ff.group_by( +// ff_norm, +// topk_indices, +// grouped_tokens, +// mixtral_config.num_local_experts, +// 0.0f, +// std::string("layers." + std::to_string(i) + ".block_sparse_moe_groupby").c_str()); // No corresponding weights // grouped_tokens[0] has dims (1024, 1, 0) - Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; Tensor one_aggregate_inputs[1] = {nullptr}; From 0eeaab414da6d3bab85d679bdab77344edc31497 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Mon, 9 Dec 2024 10:38:59 -0500 Subject: [PATCH 105/106] outputs tokens --- inference/models/mixtral.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index 7fb0f6049f..ae93c44d31 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -419,4 +419,4 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, im->register_model_weights_loader(&ff, fileloader); } -}; // namespace FlexFlow \ No newline at end of file +}; // namespace FlexFlow From 17174d2e0f81eb7874b8c7eafc8018c8a864e969 Mon Sep 17 00:00:00 2001 From: Hugo Latendresse Date: Mon, 9 Dec 2024 11:11:28 -0500 Subject: [PATCH 106/106] comments --- inference/models/mixtral.cc | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/inference/models/mixtral.cc b/inference/models/mixtral.cc index ae93c44d31..13317949b6 100644 --- a/inference/models/mixtral.cc +++ b/inference/models/mixtral.cc @@ -278,11 +278,9 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, Tensor aggregate_inputs[4 + mixtral_config.num_local_experts] = {nullptr}; Tensor one_aggregate_inputs[1] = {nullptr}; - // TODO don't use only one expert for (int expert_idx = 0; expert_idx < mixtral_config.num_local_experts; expert_idx++) { -// for (int expert_idx = 1; expert_idx < 2; expert_idx++) { Tensor w1 = ff.dense( - ff_norm, + ff_norm, // TODO should use grouped_tokens here. Dimensions of expert input will be wrong mixtral_config.intermediate_size, AC_MODE_NONE, false, @@ -296,7 +294,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, std::to_string(expert_idx) + "_w1").c_str()); Tensor w3 = ff.dense( - ff_norm, + ff_norm, // TODO should use grouped_tokens here. Dimensions of expert input will be wrong mixtral_config.intermediate_size, AC_MODE_NONE, false, @@ -331,26 +329,6 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, // Tensor topk_values_reduced = ff.reduce_sum(topk_values, {0}, true); // (2, 1, 1) // topk_values = ff.divide(topk_values, topk_values_reduced); // (2, 1, 128) - -// Tensor dummy_gate = ff.dense( // TODO try uncommenting the whole block -// ff_norm, -// mixtral_config.num_local_experts, -// AC_MODE_NONE, -// false, -// DT_NONE, -// nullptr, -// nullptr, -// nullptr, -// REG_MODE_NONE, -// 0.0f, -// std::string("layers." + std::to_string(i) + ".block_sparse_moe_gate") -// .c_str()); -// dummy_gate = ff.softmax( -// gate, -// 0, -// DT_NONE, -// std::string("dummy_gate").c_str()); -// aggregate_inputs[0] = topk_values; // (experts_per_tok, 1, 128) (3 dims confirmed) aggregate_inputs[1] = topk_indices; // (experts_per_tok, 1, 128) (3 dims confirmed) aggregate_inputs[2] = topk_values; // TODO this is a tmp fix @@ -359,7 +337,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, mlp_out = aggregate_inputs[5]; // TODO don't use just one expert // mlp_out = ff.aggregate(aggregate_inputs, //// topk_values->dims[2], -// mixtral_config.num_local_experts, // TODO don't use just one expert +// mixtral_config.num_local_experts, // 0.0f, // std::string("layers." + std::to_string(i) + ".block_sparse_moe_experts_aggregate").c_str()); @@ -367,7 +345,7 @@ void MIXTRAL::create_mixtral_model(FFModel &ff, printf("mlp_out in layer %d dims are %d %d %d %d\n",i, mlp_out->dims[0], mlp_out->dims[1], mlp_out->dims[2], mlp_out->dims[3]); assert(mlp_out->dims[0] == mixtral_config.hidden_size && "mlp_out dims[0] != hidden_size"); assert(mlp_out->dims[1] == 1 && "mlp_out dims[1] != 1"); -// assert(mlp_out->dims[2] == 128 && "mlp_out dims[2] != 128"); // TODO update with new seq len + assert(mlp_out->dims[2] == 128 && "mlp_out dims[2] != 128"); printf("seq length is now %d\n", mlp_out->dims[2]); }