From 1e2a60031ce196f30a821a26466588b0d1fd6a8f Mon Sep 17 00:00:00 2001 From: dpatel-007 Date: Thu, 30 Jan 2025 12:38:49 -0500 Subject: [PATCH 1/7] - accept model path,model list, backend type from env variables to pass values for spyre --- tests/spyre/test_spyre_basic.py | 21 +++++++++++++++++-- tests/spyre/test_spyre_embeddings.py | 21 ++++++++++++++++--- tests/spyre/test_spyre_max_prompt_length.py | 23 +++++++++++++++++++-- tests/spyre/test_spyre_seed.py | 20 ++++++++++++++++-- tests/spyre/test_spyre_tensor_parallel.py | 22 +++++++++++++++++--- tests/spyre/test_spyre_warmup_shapes.py | 22 +++++++++++++++++--- 6 files changed, 114 insertions(+), 15 deletions(-) diff --git a/tests/spyre/test_spyre_basic.py b/tests/spyre/test_spyre_basic.py index 0cd53a72a..4cfb86036 100644 --- a/tests/spyre/test_spyre_basic.py +++ b/tests/spyre/test_spyre_basic.py @@ -10,9 +10,25 @@ generate_spyre_vllm_output) from vllm import SamplingParams +import os +# get model directory path from env, if not set then default to "/models". +model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") +# get model backend from env, if not set then default to "eager" +# For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" +backend_type = os.environ.get("SYPRE_TEST_BACKEND_TYPE", "eager") +# get model names from env, if not set then default to "llama-194m" +# For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" +user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") +test_model_list, test_backend_list = [],[] -@pytest.mark.parametrize("model", ["/models/llama-194m"]) +for model in user_test_model_list.split(','): + test_model_list.append(f"{model_dir_path}/{model.strip()}") + +for backend in backend_type.split(','): + test_backend_list.append(backend.strip()) + +@pytest.mark.parametrize("model", test_model_list) @pytest.mark.parametrize("prompts", [[ "Provide a list of instructions for preparing" " chicken soup for a family of four.", "Hello", @@ -20,9 +36,10 @@ ]]) @pytest.mark.parametrize("warmup_shape", [(64, 20, 4), (64, 20, 8), (128, 20, 4), (128, 20, 8)] +# @pytest.mark.parametrize("warmup_shape", [(64, 20, 1), (128, 20, 1)] ) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("backend", - ["eager"]) #, "inductor", "sendnn_decoder"]) + test_backend_list) #, "inductor", "sendnn_decoder"]) def test_output( model: str, prompts: List[str], diff --git a/tests/spyre/test_spyre_embeddings.py b/tests/spyre/test_spyre_embeddings.py index 8c8056158..7c9bb82cb 100644 --- a/tests/spyre/test_spyre_embeddings.py +++ b/tests/spyre/test_spyre_embeddings.py @@ -1,6 +1,6 @@ """Verification of vLLM output by comparing with HF -Run `python -m pytest tests/spyre/test_spyre_embeddings.py`. +Run `python -m pytest tests/spyre/test_spyre_basic.py`. """ from typing import List, Tuple @@ -8,9 +8,24 @@ import pytest from spyre_util import (compare_embedding_results, spyre_vllm_embeddings, st_embeddings) +import os +# get model directory path from env, if not set then default to "/models". +model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") +# get model backend from env, if not set then default to "eager" +# For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" +backend_type = os.environ.get("SYPRE_TEST_BACKEND_TYPE", "eager") +# get model names from env, if not set then default to "llama-194m" +# For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" +user_test_model_list = os.environ.get("SPYRE_TEST_EMBEDDING_MODEL_LIST","all-roberta-large-v1") +test_model_list, test_backend_list = [],[] +for model in user_test_model_list.split(','): + test_model_list.append(f"{model_dir_path}/{model.strip()}") -@pytest.mark.parametrize("model", ["/models/all-roberta-large-v1"]) +for backend in backend_type.split(','): + test_backend_list.append(backend.strip()) + +@pytest.mark.parametrize("model", test_model_list) @pytest.mark.parametrize("prompts", [[ "The capital of France is Paris." "Provide a list of instructions for preparing" @@ -21,7 +36,7 @@ [(64, 4), (64, 8), (128, 4), (128, 8)]) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("backend", - ["eager"]) #, "inductor", "sendnn_decoder"]) + test_backend_list) #, "inductor", "sendnn_decoder"]) def test_output( model: str, prompts: List[str], diff --git a/tests/spyre/test_spyre_max_prompt_length.py b/tests/spyre/test_spyre_max_prompt_length.py index e2fdd9e18..291ce908a 100644 --- a/tests/spyre/test_spyre_max_prompt_length.py +++ b/tests/spyre/test_spyre_max_prompt_length.py @@ -11,9 +11,25 @@ from transformers import AutoTokenizer from vllm import SamplingParams +import os +# get model directory path from env, if not set then default to "/models". +model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") +# get model backend from env, if not set then default to "eager" +# For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" +backend_type = os.environ.get("SYPRE_TEST_BACKEND_TYPE", "eager") +# get model names from env, if not set then default to "llama-194m" +# For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" +user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") +test_model_list, test_backend_list = [],[] -@pytest.mark.parametrize("model", ["/models/llama-194m"]) +for model in user_test_model_list.split(','): + test_model_list.append(f"{model_dir_path}/{model.strip()}") + +for backend in backend_type.split(','): + test_backend_list.append(backend.strip()) + +@pytest.mark.parametrize("model", test_model_list) @pytest.mark.parametrize("prompts", [ 7 * [ "Hello", @@ -27,9 +43,12 @@ ]) @pytest.mark.parametrize("warmup_shapes", [[(64, 20, 4)], [(64, 20, 4), (128, 20, 4)]] +# ) # (prompt_length/new_tokens/batch_size) +# @pytest.mark.parametrize("warmup_shapes", +# [[(64, 20, 1)], [(128, 20, 1)]] ) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("backend", - ["eager"]) #, "inductor", "sendnn_decoder"]) + test_backend_list) #, "inductor", "sendnn_decoder"]) def test_output( model: str, prompts: List[str], diff --git a/tests/spyre/test_spyre_seed.py b/tests/spyre/test_spyre_seed.py index 01bebdce5..e5faeaf18 100644 --- a/tests/spyre/test_spyre_seed.py +++ b/tests/spyre/test_spyre_seed.py @@ -10,9 +10,25 @@ from spyre_util import generate_spyre_vllm_output from vllm import SamplingParams +import os +# get model directory path from env, if not set then default to "/models". +model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") +# get model backend from env, if not set then default to "eager" +# For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" +backend_type = os.environ.get("SYPRE_TEST_BACKEND_TYPE", "eager") +# get model names from env, if not set then default to "llama-194m" +# For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" +user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") +test_model_list, test_backend_list = [],[] -@pytest.mark.parametrize("model", ["/models/llama-194m"]) +for model in user_test_model_list.split(','): + test_model_list.append(f"{model_dir_path}/{model.strip()}") + +for backend in backend_type.split(','): + test_backend_list.append(backend.strip()) + +@pytest.mark.parametrize("model", test_model_list) @pytest.mark.parametrize("prompt", [ "Provide a list of instructions for preparing" " chicken soup for a family of four." @@ -23,7 +39,7 @@ (128, 20, 4), (128, 20, 8)] ) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("backend", - ["eager"]) #, "inductor", "sendnn_decoder"]) + test_backend_list) #, "inductor", "sendnn_decoder"]) def test_seed( model: str, prompt: str, diff --git a/tests/spyre/test_spyre_tensor_parallel.py b/tests/spyre/test_spyre_tensor_parallel.py index f6d2626fc..41d6143cd 100644 --- a/tests/spyre/test_spyre_tensor_parallel.py +++ b/tests/spyre/test_spyre_tensor_parallel.py @@ -10,20 +10,36 @@ generate_spyre_vllm_output) from vllm import SamplingParams +import os +# get model directory path from env, if not set then default to "/models". +model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") +# get model backend from env, if not set then default to "eager" +# For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" +backend_type = os.environ.get("SYPRE_TEST_BACKEND_TYPE", "eager") +# get model names from env, if not set then default to "llama-194m" +# For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" +user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") +test_model_list, test_backend_list = [],[] -@pytest.mark.parametrize("model", ["/models/llama-194m"]) +for model in user_test_model_list.split(','): + test_model_list.append(f"{model_dir_path.strip()}/{model.strip()}") + +for backend in backend_type.split(','): + test_backend_list.append(backend.strip()) + +@pytest.mark.parametrize("model", test_model_list) @pytest.mark.parametrize("prompts", [[ "Provide a list of instructions for preparing" " chicken soup for a family of four.", "Hello", "What is the weather today like?", "Who are you?" ]]) -@pytest.mark.parametrize("warmup_shapes", [[(64, 20, 4)]] +@pytest.mark.parametrize("warmup_shapes", [[(64, 20, 1)]] ) #,[(64,20,8)],[(128,20,4)],[(128,20,8)]]) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("tp_size", [2]) @pytest.mark.parametrize("backend", - ["eager"]) #, "inductor", "sendnn_decoder"]) + test_backend_list) #, "inductor", "sendnn_decoder"]) def test_output( model: str, prompts: List[str], diff --git a/tests/spyre/test_spyre_warmup_shapes.py b/tests/spyre/test_spyre_warmup_shapes.py index 675b9188f..beb3621e5 100644 --- a/tests/spyre/test_spyre_warmup_shapes.py +++ b/tests/spyre/test_spyre_warmup_shapes.py @@ -10,9 +10,25 @@ generate_spyre_vllm_output) from vllm import SamplingParams +import os +# get model directory path from env, if not set then default to "/models". +model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") +# get model backend from env, if not set then default to "eager" +# For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" +backend_type = os.environ.get("SYPRE_TEST_BACKEND_TYPE", "eager") +# get model names from env, if not set then default to "llama-194m" +# For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" +user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") +test_model_list, test_backend_list = [],[] -@pytest.mark.parametrize("model", ["/models/llama-194m"]) +for model in user_test_model_list.split(','): + test_model_list.append(f"{model_dir_path.strip()}/{model.strip()}") + +for backend in backend_type.split(','): + test_backend_list.append(backend.strip()) + +@pytest.mark.parametrize("model", test_model_list) @pytest.mark.parametrize("prompts", [ 7 * [ "Hello", @@ -25,9 +41,9 @@ ] ]) @pytest.mark.parametrize("warmup_shapes", [[(64, 20, 8), (128, 20, 4)]] - ) # (prompt_length/new_tokens/batch_size) + ) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("backend", - ["eager"]) #, "inductor", "sendnn_decoder"]) + test_backend_list) #, "inductor", "sendnn_decoder"]) def test_output( model: str, prompts: List[str], From faa9ff4d224bff7158cdffd1903503595da3f953 Mon Sep 17 00:00:00 2001 From: dpatel-007 Date: Mon, 3 Feb 2025 09:27:07 -0500 Subject: [PATCH 2/7] fix typo --- tests/spyre/test_spyre_basic.py | 2 +- tests/spyre/test_spyre_embeddings.py | 2 +- tests/spyre/test_spyre_max_prompt_length.py | 2 +- tests/spyre/test_spyre_seed.py | 2 +- tests/spyre/test_spyre_tensor_parallel.py | 2 +- tests/spyre/test_spyre_warmup_shapes.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/spyre/test_spyre_basic.py b/tests/spyre/test_spyre_basic.py index 4cfb86036..2491ce9bf 100644 --- a/tests/spyre/test_spyre_basic.py +++ b/tests/spyre/test_spyre_basic.py @@ -16,7 +16,7 @@ model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") # get model backend from env, if not set then default to "eager" # For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" -backend_type = os.environ.get("SYPRE_TEST_BACKEND_TYPE", "eager") +backend_type = os.environ.get("SPYRE_TEST_BACKEND_TYPE", "eager") # get model names from env, if not set then default to "llama-194m" # For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") diff --git a/tests/spyre/test_spyre_embeddings.py b/tests/spyre/test_spyre_embeddings.py index 7c9bb82cb..02bad635b 100644 --- a/tests/spyre/test_spyre_embeddings.py +++ b/tests/spyre/test_spyre_embeddings.py @@ -13,7 +13,7 @@ model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") # get model backend from env, if not set then default to "eager" # For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" -backend_type = os.environ.get("SYPRE_TEST_BACKEND_TYPE", "eager") +backend_type = os.environ.get("SPYRE_TEST_BACKEND_TYPE", "eager") # get model names from env, if not set then default to "llama-194m" # For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" user_test_model_list = os.environ.get("SPYRE_TEST_EMBEDDING_MODEL_LIST","all-roberta-large-v1") diff --git a/tests/spyre/test_spyre_max_prompt_length.py b/tests/spyre/test_spyre_max_prompt_length.py index 291ce908a..d223d1a31 100644 --- a/tests/spyre/test_spyre_max_prompt_length.py +++ b/tests/spyre/test_spyre_max_prompt_length.py @@ -17,7 +17,7 @@ model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") # get model backend from env, if not set then default to "eager" # For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" -backend_type = os.environ.get("SYPRE_TEST_BACKEND_TYPE", "eager") +backend_type = os.environ.get("SPYRE_TEST_BACKEND_TYPE", "eager") # get model names from env, if not set then default to "llama-194m" # For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") diff --git a/tests/spyre/test_spyre_seed.py b/tests/spyre/test_spyre_seed.py index e5faeaf18..94a3b29d7 100644 --- a/tests/spyre/test_spyre_seed.py +++ b/tests/spyre/test_spyre_seed.py @@ -16,7 +16,7 @@ model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") # get model backend from env, if not set then default to "eager" # For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" -backend_type = os.environ.get("SYPRE_TEST_BACKEND_TYPE", "eager") +backend_type = os.environ.get("SPYRE_TEST_BACKEND_TYPE", "eager") # get model names from env, if not set then default to "llama-194m" # For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") diff --git a/tests/spyre/test_spyre_tensor_parallel.py b/tests/spyre/test_spyre_tensor_parallel.py index 41d6143cd..952463245 100644 --- a/tests/spyre/test_spyre_tensor_parallel.py +++ b/tests/spyre/test_spyre_tensor_parallel.py @@ -16,7 +16,7 @@ model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") # get model backend from env, if not set then default to "eager" # For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" -backend_type = os.environ.get("SYPRE_TEST_BACKEND_TYPE", "eager") +backend_type = os.environ.get("SPYRE_TEST_BACKEND_TYPE", "eager") # get model names from env, if not set then default to "llama-194m" # For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") diff --git a/tests/spyre/test_spyre_warmup_shapes.py b/tests/spyre/test_spyre_warmup_shapes.py index beb3621e5..c4d031e80 100644 --- a/tests/spyre/test_spyre_warmup_shapes.py +++ b/tests/spyre/test_spyre_warmup_shapes.py @@ -16,7 +16,7 @@ model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") # get model backend from env, if not set then default to "eager" # For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" -backend_type = os.environ.get("SYPRE_TEST_BACKEND_TYPE", "eager") +backend_type = os.environ.get("SPYRE_TEST_BACKEND_TYPE", "eager") # get model names from env, if not set then default to "llama-194m" # For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") From 705669e1e165656f79456f1c7de4e62d63ba0dfc Mon Sep 17 00:00:00 2001 From: dpatel-007 Date: Mon, 10 Feb 2025 08:47:55 -0500 Subject: [PATCH 3/7] fix file name in run command --- tests/spyre/test_spyre_embeddings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/spyre/test_spyre_embeddings.py b/tests/spyre/test_spyre_embeddings.py index 02bad635b..5e53b4915 100644 --- a/tests/spyre/test_spyre_embeddings.py +++ b/tests/spyre/test_spyre_embeddings.py @@ -1,6 +1,6 @@ """Verification of vLLM output by comparing with HF -Run `python -m pytest tests/spyre/test_spyre_basic.py`. +Run `python -m pytest tests/spyre/test_spyre_embeddings.py`. """ from typing import List, Tuple From 59a86dac78e88bef436a432aab1593012d9cc46c Mon Sep 17 00:00:00 2001 From: dpatel-007 Date: Tue, 11 Feb 2025 19:32:39 -0500 Subject: [PATCH 4/7] - update variable names - fix comments --- tests/spyre/test_spyre_basic.py | 11 +++++------ tests/spyre/test_spyre_embeddings.py | 12 ++++++------ tests/spyre/test_spyre_max_prompt_length.py | 15 ++++++--------- tests/spyre/test_spyre_seed.py | 10 +++++----- tests/spyre/test_spyre_tensor_parallel.py | 10 +++++----- tests/spyre/test_spyre_warmup_shapes.py | 10 +++++----- 6 files changed, 32 insertions(+), 36 deletions(-) diff --git a/tests/spyre/test_spyre_basic.py b/tests/spyre/test_spyre_basic.py index 2491ce9bf..b9acc765e 100644 --- a/tests/spyre/test_spyre_basic.py +++ b/tests/spyre/test_spyre_basic.py @@ -15,17 +15,17 @@ # get model directory path from env, if not set then default to "/models". model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") # get model backend from env, if not set then default to "eager" -# For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" -backend_type = os.environ.get("SPYRE_TEST_BACKEND_TYPE", "eager") +# For multiple values, export SPYRE_TEST_BACKEND_LIST="eager,inductor,sendnn_decoder" +backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") # get model names from env, if not set then default to "llama-194m" -# For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" +# For multiple values, export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") test_model_list, test_backend_list = [],[] for model in user_test_model_list.split(','): test_model_list.append(f"{model_dir_path}/{model.strip()}") -for backend in backend_type.split(','): +for backend in backend_list.split(','): test_backend_list.append(backend.strip()) @pytest.mark.parametrize("model", test_model_list) @@ -36,10 +36,9 @@ ]]) @pytest.mark.parametrize("warmup_shape", [(64, 20, 4), (64, 20, 8), (128, 20, 4), (128, 20, 8)] -# @pytest.mark.parametrize("warmup_shape", [(64, 20, 1), (128, 20, 1)] ) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("backend", - test_backend_list) #, "inductor", "sendnn_decoder"]) + test_backend_list) def test_output( model: str, prompts: List[str], diff --git a/tests/spyre/test_spyre_embeddings.py b/tests/spyre/test_spyre_embeddings.py index 5e53b4915..40b3e08e6 100644 --- a/tests/spyre/test_spyre_embeddings.py +++ b/tests/spyre/test_spyre_embeddings.py @@ -12,17 +12,17 @@ # get model directory path from env, if not set then default to "/models". model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") # get model backend from env, if not set then default to "eager" -# For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" -backend_type = os.environ.get("SPYRE_TEST_BACKEND_TYPE", "eager") -# get model names from env, if not set then default to "llama-194m" -# For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" +# For multiple values, export SPYRE_TEST_BACKEND_LIST="eager,inductor,sendnn_decoder" +backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") +# get model names from env, if not set then default to "all-roberta-large-v1" +# For multiple values, export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" user_test_model_list = os.environ.get("SPYRE_TEST_EMBEDDING_MODEL_LIST","all-roberta-large-v1") test_model_list, test_backend_list = [],[] for model in user_test_model_list.split(','): test_model_list.append(f"{model_dir_path}/{model.strip()}") -for backend in backend_type.split(','): +for backend in backend_list.split(','): test_backend_list.append(backend.strip()) @pytest.mark.parametrize("model", test_model_list) @@ -36,7 +36,7 @@ [(64, 4), (64, 8), (128, 4), (128, 8)]) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("backend", - test_backend_list) #, "inductor", "sendnn_decoder"]) + test_backend_list) def test_output( model: str, prompts: List[str], diff --git a/tests/spyre/test_spyre_max_prompt_length.py b/tests/spyre/test_spyre_max_prompt_length.py index d223d1a31..9739caf0e 100644 --- a/tests/spyre/test_spyre_max_prompt_length.py +++ b/tests/spyre/test_spyre_max_prompt_length.py @@ -16,17 +16,17 @@ # get model directory path from env, if not set then default to "/models". model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") # get model backend from env, if not set then default to "eager" -# For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" -backend_type = os.environ.get("SPYRE_TEST_BACKEND_TYPE", "eager") +# For multiple values, export SPYRE_TEST_BACKEND_LIST="eager,inductor,sendnn_decoder" +backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") # get model names from env, if not set then default to "llama-194m" -# For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" +# For multiple values, export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") test_model_list, test_backend_list = [],[] for model in user_test_model_list.split(','): test_model_list.append(f"{model_dir_path}/{model.strip()}") -for backend in backend_type.split(','): +for backend in backend_list.split(','): test_backend_list.append(backend.strip()) @pytest.mark.parametrize("model", test_model_list) @@ -43,12 +43,9 @@ ]) @pytest.mark.parametrize("warmup_shapes", [[(64, 20, 4)], [(64, 20, 4), (128, 20, 4)]] -# ) # (prompt_length/new_tokens/batch_size) -# @pytest.mark.parametrize("warmup_shapes", -# [[(64, 20, 1)], [(128, 20, 1)]] - ) # (prompt_length/new_tokens/batch_size) + ) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("backend", - test_backend_list) #, "inductor", "sendnn_decoder"]) + test_backend_list) def test_output( model: str, prompts: List[str], diff --git a/tests/spyre/test_spyre_seed.py b/tests/spyre/test_spyre_seed.py index 94a3b29d7..a78fd992d 100644 --- a/tests/spyre/test_spyre_seed.py +++ b/tests/spyre/test_spyre_seed.py @@ -15,17 +15,17 @@ # get model directory path from env, if not set then default to "/models". model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") # get model backend from env, if not set then default to "eager" -# For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" -backend_type = os.environ.get("SPYRE_TEST_BACKEND_TYPE", "eager") +# For multiple values, export SPYRE_TEST_BACKEND_LIST="eager,inductor,sendnn_decoder" +backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") # get model names from env, if not set then default to "llama-194m" -# For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" +# For multiple values, export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") test_model_list, test_backend_list = [],[] for model in user_test_model_list.split(','): test_model_list.append(f"{model_dir_path}/{model.strip()}") -for backend in backend_type.split(','): +for backend in backend_list.split(','): test_backend_list.append(backend.strip()) @pytest.mark.parametrize("model", test_model_list) @@ -39,7 +39,7 @@ (128, 20, 4), (128, 20, 8)] ) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("backend", - test_backend_list) #, "inductor", "sendnn_decoder"]) + test_backend_list) def test_seed( model: str, prompt: str, diff --git a/tests/spyre/test_spyre_tensor_parallel.py b/tests/spyre/test_spyre_tensor_parallel.py index 952463245..be1135890 100644 --- a/tests/spyre/test_spyre_tensor_parallel.py +++ b/tests/spyre/test_spyre_tensor_parallel.py @@ -15,17 +15,17 @@ # get model directory path from env, if not set then default to "/models". model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") # get model backend from env, if not set then default to "eager" -# For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" -backend_type = os.environ.get("SPYRE_TEST_BACKEND_TYPE", "eager") +# For multiple values, export SPYRE_TEST_BACKEND_LIST="eager,inductor,sendnn_decoder" +backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") # get model names from env, if not set then default to "llama-194m" -# For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" +# For multiple values, export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") test_model_list, test_backend_list = [],[] for model in user_test_model_list.split(','): test_model_list.append(f"{model_dir_path.strip()}/{model.strip()}") -for backend in backend_type.split(','): +for backend in backend_list.split(','): test_backend_list.append(backend.strip()) @pytest.mark.parametrize("model", test_model_list) @@ -39,7 +39,7 @@ # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("tp_size", [2]) @pytest.mark.parametrize("backend", - test_backend_list) #, "inductor", "sendnn_decoder"]) + test_backend_list) def test_output( model: str, prompts: List[str], diff --git a/tests/spyre/test_spyre_warmup_shapes.py b/tests/spyre/test_spyre_warmup_shapes.py index c4d031e80..48ecff901 100644 --- a/tests/spyre/test_spyre_warmup_shapes.py +++ b/tests/spyre/test_spyre_warmup_shapes.py @@ -15,17 +15,17 @@ # get model directory path from env, if not set then default to "/models". model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") # get model backend from env, if not set then default to "eager" -# For multiple values, export SPYRE_TEST_MODEL_DIR="eager,inductor" -backend_type = os.environ.get("SPYRE_TEST_BACKEND_TYPE", "eager") +# For multiple values, export SPYRE_TEST_BACKEND_LIST="eager,inductor,sendnn_decoder" +backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") # get model names from env, if not set then default to "llama-194m" -# For multiple values, export SPYRE_TEST_MODEL_DIR="llama-194m,all-roberta-large-v1" +# For multiple values, export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") test_model_list, test_backend_list = [],[] for model in user_test_model_list.split(','): test_model_list.append(f"{model_dir_path.strip()}/{model.strip()}") -for backend in backend_type.split(','): +for backend in backend_list.split(','): test_backend_list.append(backend.strip()) @pytest.mark.parametrize("model", test_model_list) @@ -43,7 +43,7 @@ @pytest.mark.parametrize("warmup_shapes", [[(64, 20, 8), (128, 20, 4)]] ) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("backend", - test_backend_list) #, "inductor", "sendnn_decoder"]) + test_backend_list) def test_output( model: str, prompts: List[str], From 6e3083b86b9b9ac47a69ecf19899982b2c4fcb02 Mon Sep 17 00:00:00 2001 From: dpatel-007 Date: Sat, 15 Feb 2025 18:47:42 -0500 Subject: [PATCH 5/7] Signed-off-by: Dhruval Patel - fixed E501 erros - created functions in utils and removed repeated code --- tests/spyre/spyre_util.py | 40 +++++++++++ tests/spyre/test_spyre_basic.py | 61 ++++++++--------- tests/spyre/test_spyre_embeddings.py | 76 ++++++++++----------- tests/spyre/test_spyre_max_new_tokens.py | 45 +++++++----- tests/spyre/test_spyre_max_prompt_length.py | 62 +++++++---------- tests/spyre/test_spyre_seed.py | 52 ++++++-------- tests/spyre/test_spyre_tensor_parallel.py | 63 ++++++++--------- tests/spyre/test_spyre_warmup_shapes.py | 53 ++++++-------- 8 files changed, 228 insertions(+), 224 deletions(-) diff --git a/tests/spyre/spyre_util.py b/tests/spyre/spyre_util.py index c2985453c..f414cc864 100644 --- a/tests/spyre/spyre_util.py +++ b/tests/spyre/spyre_util.py @@ -281,3 +281,43 @@ def compare_embedding_results(model: str, prompts: List[str], vllm_result["embeddings"]) assert math.isclose(sim, 1.0, rel_tol=0.05) + + +# get model directory path from env, if not set then default to "/models". +def get_spyre_model_dir_path(): + model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") + return model_dir_path + + +# get model backend from env, if not set then default to "eager" +# For multiple values: +# export SPYRE_TEST_BACKEND_LIST="eager, inductor, sendnn_decoder" +def get_spyre_backend_list(isEmbeddings=False): + test_backend_list = [] + user_backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") + + # set default to sendnn if testing embeddings + if isEmbeddings: + user_backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "sendnn") + for backend in user_backend_list.split(","): + test_backend_list.append(backend.strip()) + return test_backend_list + + +# get model names from env, if not set then default to "llama-194m" +# For multiple values: +# export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" +def get_spyre_model_list(isEmbeddings=False): + spyre_model_dir_path = get_spyre_model_dir_path() + test_model_list = [] + user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST", + "llama-194m") + + # set default to bert if testing embeddings + if isEmbeddings: + user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST", + "all-roberta-large-v1") + + for model in user_test_model_list.split(","): + test_model_list.append(f"{spyre_model_dir_path}/{model.strip()}") + return test_model_list diff --git a/tests/spyre/test_spyre_basic.py b/tests/spyre/test_spyre_basic.py index b9acc765e..6ab4db02f 100644 --- a/tests/spyre/test_spyre_basic.py +++ b/tests/spyre/test_spyre_basic.py @@ -7,38 +7,27 @@ import pytest from spyre_util import (compare_results, generate_hf_output, - generate_spyre_vllm_output) + generate_spyre_vllm_output, get_spyre_backend_list, + get_spyre_model_list) from vllm import SamplingParams -import os -# get model directory path from env, if not set then default to "/models". -model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") -# get model backend from env, if not set then default to "eager" -# For multiple values, export SPYRE_TEST_BACKEND_LIST="eager,inductor,sendnn_decoder" -backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") -# get model names from env, if not set then default to "llama-194m" -# For multiple values, export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" -user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") -test_model_list, test_backend_list = [],[] -for model in user_test_model_list.split(','): - test_model_list.append(f"{model_dir_path}/{model.strip()}") - -for backend in backend_list.split(','): - test_backend_list.append(backend.strip()) - -@pytest.mark.parametrize("model", test_model_list) -@pytest.mark.parametrize("prompts", [[ - "Provide a list of instructions for preparing" - " chicken soup for a family of four.", "Hello", - "What is the weather today like?", "Who are you?" -]]) +@pytest.mark.parametrize("model", get_spyre_model_list()) +@pytest.mark.parametrize( + "prompts", + [[ + "Provide a list of instructions for preparing" + " chicken soup for a family of four.", + "Hello", + "What is the weather today like?", + "Who are you?", + ]], +) @pytest.mark.parametrize("warmup_shape", [(64, 20, 4), (64, 20, 8), (128, 20, 4), (128, 20, 8)] ) # (prompt_length/new_tokens/batch_size) -@pytest.mark.parametrize("backend", - test_backend_list) +@pytest.mark.parametrize("backend", get_spyre_backend_list()) def test_output( model: str, prompts: List[str], @@ -64,7 +53,8 @@ def test_output( max_tokens=max_new_tokens, temperature=0, logprobs=0, # return logprobs of generated tokens only - ignore_eos=True) + ignore_eos=True, + ) vllm_results = generate_spyre_vllm_output( model=model, @@ -74,16 +64,19 @@ def test_output( block_size=2048, sampling_params=vllm_sampling_params, tensor_parallel_size=1, - backend=backend) + backend=backend, + ) hf_results = generate_hf_output(model=model, prompts=prompts, max_new_tokens=max_new_tokens) - compare_results(model=model, - prompts=prompts, - warmup_shapes=[warmup_shape], - tensor_parallel_size=1, - backend=backend, - vllm_results=vllm_results, - hf_results=hf_results) + compare_results( + model=model, + prompts=prompts, + warmup_shapes=[warmup_shape], + tensor_parallel_size=1, + backend=backend, + vllm_results=vllm_results, + hf_results=hf_results, + ) diff --git a/tests/spyre/test_spyre_embeddings.py b/tests/spyre/test_spyre_embeddings.py index 40b3e08e6..08f0138ef 100644 --- a/tests/spyre/test_spyre_embeddings.py +++ b/tests/spyre/test_spyre_embeddings.py @@ -7,36 +7,26 @@ import pytest from spyre_util import (compare_embedding_results, spyre_vllm_embeddings, - st_embeddings) -import os -# get model directory path from env, if not set then default to "/models". -model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") -# get model backend from env, if not set then default to "eager" -# For multiple values, export SPYRE_TEST_BACKEND_LIST="eager,inductor,sendnn_decoder" -backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") -# get model names from env, if not set then default to "all-roberta-large-v1" -# For multiple values, export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" -user_test_model_list = os.environ.get("SPYRE_TEST_EMBEDDING_MODEL_LIST","all-roberta-large-v1") -test_model_list, test_backend_list = [],[] - -for model in user_test_model_list.split(','): - test_model_list.append(f"{model_dir_path}/{model.strip()}") - -for backend in backend_list.split(','): - test_backend_list.append(backend.strip()) - -@pytest.mark.parametrize("model", test_model_list) -@pytest.mark.parametrize("prompts", [[ - "The capital of France is Paris." - "Provide a list of instructions for preparing" - " chicken soup for a family of four.", "Hello", - "What is the weather today like?", "Who are you?" -]]) + st_embeddings, get_spyre_backend_list, + get_spyre_model_list) + + +@pytest.mark.parametrize("model", get_spyre_model_list(isEmbeddings=True)) +@pytest.mark.parametrize( + "prompts", + [[ + "The capital of France is Paris." + "Provide a list of instructions for preparing" + " chicken soup for a family of four.", + "Hello", + "What is the weather today like?", + "Who are you?", + ]], +) @pytest.mark.parametrize("warmup_shape", [(64, 4), (64, 8), (128, 4), (128, 8)]) # (prompt_length/new_tokens/batch_size) -@pytest.mark.parametrize("backend", - test_backend_list) +@pytest.mark.parametrize("backend", get_spyre_backend_list(isEmbeddings=True)) def test_output( model: str, prompts: List[str], @@ -50,20 +40,24 @@ def test_output( are verified to be identical for vLLM and SentenceTransformers. ''' - vllm_results = spyre_vllm_embeddings(model=model, - prompts=prompts, - warmup_shapes=[warmup_shape], - max_model_len=256, - block_size=256, - tensor_parallel_size=1, - backend=backend) + vllm_results = spyre_vllm_embeddings( + model=model, + prompts=prompts, + warmup_shapes=[warmup_shape], + max_model_len=256, + block_size=256, + tensor_parallel_size=1, + backend=backend, + ) hf_results = st_embeddings(model=model, prompts=prompts) - compare_embedding_results(model=model, - prompts=prompts, - warmup_shapes=[warmup_shape], - tensor_parallel_size=1, - backend=backend, - vllm_results=vllm_results, - hf_results=hf_results) + compare_embedding_results( + model=model, + prompts=prompts, + warmup_shapes=[warmup_shape], + tensor_parallel_size=1, + backend=backend, + vllm_results=vllm_results, + hf_results=hf_results, + ) diff --git a/tests/spyre/test_spyre_max_new_tokens.py b/tests/spyre/test_spyre_max_new_tokens.py index fccb67084..9a301fb50 100644 --- a/tests/spyre/test_spyre_max_new_tokens.py +++ b/tests/spyre/test_spyre_max_new_tokens.py @@ -6,8 +6,11 @@ from typing import List, Tuple import pytest -from spyre_util import (compare_results, generate_hf_output, - generate_spyre_vllm_output) +from spyre_util import ( + compare_results, + generate_hf_output, + generate_spyre_vllm_output, +) from vllm import SamplingParams @@ -22,14 +25,19 @@ @pytest.mark.parametrize("model", ["/models/llama-194m"]) -@pytest.mark.parametrize("prompts", [[prompt1, prompt2, prompt2, prompt2], - [prompt2, prompt2, prompt2, prompt1], - [prompt2, prompt2, prompt2, prompt2]]) +@pytest.mark.parametrize( + "prompts", + [ + [prompt1, prompt2, prompt2, prompt2], + [prompt2, prompt2, prompt2, prompt1], + [prompt2, prompt2, prompt2, prompt2], + ], +) @pytest.mark.parametrize("stop_last", [True, False]) @pytest.mark.parametrize("warmup_shape", [(64, 10, 4)] ) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("backend", - ["eager"]) #, "inductor", "sendnn_decoder"]) + ["eager"]) # , "inductor", "sendnn_decoder"]) def test_output( model: str, prompts: List[str], @@ -57,13 +65,15 @@ def test_output( max_tokens=max_new_tokens_warmup, temperature=0, logprobs=0, # return logprobs of generated tokens only - ignore_eos=False) + ignore_eos=False, + ) vllm_sampling_params_early_stop = SamplingParams( max_tokens=max_new_tokens_early_stop, temperature=0, logprobs=0, # return logprobs of generated tokens only - ignore_eos=False) + ignore_eos=False, + ) vllm_sampling_params = [vllm_sampling_params_normal] * 3 hf_max_new_tokens = [max_new_tokens_warmup] * 3 @@ -87,16 +97,19 @@ def test_output( block_size=2048, sampling_params=vllm_sampling_params, tensor_parallel_size=1, - backend=backend) + backend=backend, + ) hf_results = generate_hf_output(model=model, prompts=prompts, max_new_tokens=hf_max_new_tokens) - compare_results(model=model, - prompts=prompts, - warmup_shapes=[warmup_shape], - tensor_parallel_size=1, - backend=backend, - vllm_results=vllm_results, - hf_results=hf_results) + compare_results( + model=model, + prompts=prompts, + warmup_shapes=[warmup_shape], + tensor_parallel_size=1, + backend=backend, + vllm_results=vllm_results, + hf_results=hf_results, + ) diff --git a/tests/spyre/test_spyre_max_prompt_length.py b/tests/spyre/test_spyre_max_prompt_length.py index 9739caf0e..654da41a2 100644 --- a/tests/spyre/test_spyre_max_prompt_length.py +++ b/tests/spyre/test_spyre_max_prompt_length.py @@ -7,29 +7,15 @@ import pytest from spyre_util import (compare_results, generate_hf_output, - generate_spyre_vllm_output) + generate_spyre_vllm_output, get_spyre_backend_list, + get_spyre_model_list) + from transformers import AutoTokenizer from vllm import SamplingParams -import os - -# get model directory path from env, if not set then default to "/models". -model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") -# get model backend from env, if not set then default to "eager" -# For multiple values, export SPYRE_TEST_BACKEND_LIST="eager,inductor,sendnn_decoder" -backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") -# get model names from env, if not set then default to "llama-194m" -# For multiple values, export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" -user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") -test_model_list, test_backend_list = [],[] -for model in user_test_model_list.split(','): - test_model_list.append(f"{model_dir_path}/{model.strip()}") -for backend in backend_list.split(','): - test_backend_list.append(backend.strip()) - -@pytest.mark.parametrize("model", test_model_list) +@pytest.mark.parametrize("model", get_spyre_model_list()) @pytest.mark.parametrize("prompts", [ 7 * [ "Hello", @@ -38,14 +24,14 @@ " to the user. Provide a list of instructions for preparing chicken " "soup for a family of four. Indicate if the weather forecast looks " "good for today. Explain in a brief summary comprised of at most 50" - " words what you are." + " words what you are.", ] -]) + ], +) @pytest.mark.parametrize("warmup_shapes", [[(64, 20, 4)], [(64, 20, 4), (128, 20, 4)]] - ) # (prompt_length/new_tokens/batch_size) -@pytest.mark.parametrize("backend", - test_backend_list) + ) # (prompt_length/new_tokens/batch_size) +@pytest.mark.parametrize("backend", get_spyre_backend_list()) def test_output( model: str, prompts: List[str], @@ -78,7 +64,8 @@ def test_output( max_tokens=max_new_tokens, temperature=0, logprobs=0, # return logprobs of generated tokens only - ignore_eos=True) + ignore_eos=True, + ) vllm_results = generate_spyre_vllm_output( model=model, @@ -88,7 +75,8 @@ def test_output( block_size=2048, sampling_params=vllm_sampling_params, tensor_parallel_size=1, - backend=backend) + backend=backend, + ) hf_results = generate_hf_output(model=model, prompts=prompts, @@ -102,16 +90,18 @@ def test_output( hf_input_tokens = hf_tokenizer(prompt, return_tensors="pt").input_ids if len(hf_input_tokens[0]) > max_prompt_length: hf_results[prompt_index] = { - 'text': '', - 'token_ids': (), - 'tokens': (), - 'logprobs': () + "text": "", + "token_ids": (), + "tokens": (), + "logprobs": (), } - compare_results(model=model, - prompts=prompts, - warmup_shapes=warmup_shapes, - tensor_parallel_size=1, - backend=backend, - vllm_results=vllm_results, - hf_results=hf_results) + compare_results( + model=model, + prompts=prompts, + warmup_shapes=warmup_shapes, + tensor_parallel_size=1, + backend=backend, + vllm_results=vllm_results, + hf_results=hf_results, + ) diff --git a/tests/spyre/test_spyre_seed.py b/tests/spyre/test_spyre_seed.py index a78fd992d..6f6939a0d 100644 --- a/tests/spyre/test_spyre_seed.py +++ b/tests/spyre/test_spyre_seed.py @@ -7,39 +7,26 @@ from typing import Tuple import pytest -from spyre_util import generate_spyre_vllm_output +from spyre_util import (generate_spyre_vllm_output, get_spyre_backend_list, + get_spyre_model_list) from vllm import SamplingParams -import os -# get model directory path from env, if not set then default to "/models". -model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") -# get model backend from env, if not set then default to "eager" -# For multiple values, export SPYRE_TEST_BACKEND_LIST="eager,inductor,sendnn_decoder" -backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") -# get model names from env, if not set then default to "llama-194m" -# For multiple values, export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" -user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") -test_model_list, test_backend_list = [],[] -for model in user_test_model_list.split(','): - test_model_list.append(f"{model_dir_path}/{model.strip()}") - -for backend in backend_list.split(','): - test_backend_list.append(backend.strip()) - -@pytest.mark.parametrize("model", test_model_list) -@pytest.mark.parametrize("prompt", [ - "Provide a list of instructions for preparing" - " chicken soup for a family of four." -]) +@pytest.mark.parametrize("model", get_spyre_model_list()) +@pytest.mark.parametrize( + "prompt", + [ + "Provide a list of instructions for preparing" + " chicken soup for a family of four." + ], +) @pytest.mark.parametrize("temperature", [0.1, 1.0]) @pytest.mark.parametrize("seed", [42]) @pytest.mark.parametrize("warmup_shape", [(64, 20, 4), (64, 20, 8), (128, 20, 4), (128, 20, 8)] ) # (prompt_length/new_tokens/batch_size) -@pytest.mark.parametrize("backend", - test_backend_list) +@pytest.mark.parametrize("backend", get_spyre_backend_list()) def test_seed( model: str, prompt: str, @@ -65,7 +52,8 @@ def test_seed( temperature=temperature, logprobs=0, # return logprobs of generated tokens only ignore_eos=True, - seed=seed) + seed=seed, + ) vllm_results = generate_spyre_vllm_output( model=model, @@ -75,17 +63,21 @@ def test_seed( block_size=2048, sampling_params=vllm_sampling_params, tensor_parallel_size=1, - backend=backend) + backend=backend, + ) # compare all generated outputs against the first generated output for vllm_result in vllm_results: - assert vllm_result['text'] == vllm_results[0]['text'] + assert vllm_result["text"] == vllm_results[0]["text"] # compare logprobs for all tokens between # the current and the first sequence - assert len(vllm_result['logprobs']) == len(vllm_results[0]['logprobs']) + assert len(vllm_result["logprobs"]) == len(vllm_results[0]["logprobs"]) for token_id, logprob, token_id_0, logprob_0 in zip( - vllm_result['token_ids'], vllm_result['logprobs'], - vllm_results[0]['token_ids'], vllm_results[0]['logprobs']): + vllm_result["token_ids"], + vllm_result["logprobs"], + vllm_results[0]["token_ids"], + vllm_results[0]["logprobs"], + ): assert token_id == token_id_0 assert math.isclose(logprob, logprob_0, rel_tol=0.1) diff --git a/tests/spyre/test_spyre_tensor_parallel.py b/tests/spyre/test_spyre_tensor_parallel.py index be1135890..a53721039 100644 --- a/tests/spyre/test_spyre_tensor_parallel.py +++ b/tests/spyre/test_spyre_tensor_parallel.py @@ -7,39 +7,28 @@ import pytest from spyre_util import (compare_results, generate_hf_output, - generate_spyre_vllm_output) + generate_spyre_vllm_output, get_spyre_backend_list, + get_spyre_model_list) from vllm import SamplingParams -import os -# get model directory path from env, if not set then default to "/models". -model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") -# get model backend from env, if not set then default to "eager" -# For multiple values, export SPYRE_TEST_BACKEND_LIST="eager,inductor,sendnn_decoder" -backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") -# get model names from env, if not set then default to "llama-194m" -# For multiple values, export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" -user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") -test_model_list, test_backend_list = [],[] -for model in user_test_model_list.split(','): - test_model_list.append(f"{model_dir_path.strip()}/{model.strip()}") - -for backend in backend_list.split(','): - test_backend_list.append(backend.strip()) - -@pytest.mark.parametrize("model", test_model_list) -@pytest.mark.parametrize("prompts", [[ - "Provide a list of instructions for preparing" - " chicken soup for a family of four.", "Hello", - "What is the weather today like?", "Who are you?" -]]) +@pytest.mark.parametrize("model", get_spyre_model_list()) +@pytest.mark.parametrize( + "prompts", + [[ + "Provide a list of instructions for preparing" + " chicken soup for a family of four.", + "Hello", + "What is the weather today like?", + "Who are you?", + ]], +) @pytest.mark.parametrize("warmup_shapes", [[(64, 20, 1)]] - ) #,[(64,20,8)],[(128,20,4)],[(128,20,8)]]) + ) # ,[(64,20,8)],[(128,20,4)],[(128,20,8)]]) # (prompt_length/new_tokens/batch_size) @pytest.mark.parametrize("tp_size", [2]) -@pytest.mark.parametrize("backend", - test_backend_list) +@pytest.mark.parametrize("backend", get_spyre_backend_list()) def test_output( model: str, prompts: List[str], @@ -67,7 +56,8 @@ def test_output( max_tokens=max_new_tokens, temperature=0, logprobs=0, # return logprobs of generated tokens only - ignore_eos=True) + ignore_eos=True, + ) vllm_results = generate_spyre_vllm_output( model=model, @@ -77,16 +67,19 @@ def test_output( block_size=2048, sampling_params=vllm_sampling_params, tensor_parallel_size=tp_size, - backend=backend) + backend=backend, + ) hf_results = generate_hf_output(model=model, prompts=prompts, max_new_tokens=max_new_tokens) - compare_results(model=model, - prompts=prompts, - warmup_shapes=warmup_shapes, - tensor_parallel_size=tp_size, - backend=backend, - vllm_results=vllm_results, - hf_results=hf_results) + compare_results( + model=model, + prompts=prompts, + warmup_shapes=warmup_shapes, + tensor_parallel_size=tp_size, + backend=backend, + vllm_results=vllm_results, + hf_results=hf_results, + ) diff --git a/tests/spyre/test_spyre_warmup_shapes.py b/tests/spyre/test_spyre_warmup_shapes.py index 48ecff901..2aaa8d507 100644 --- a/tests/spyre/test_spyre_warmup_shapes.py +++ b/tests/spyre/test_spyre_warmup_shapes.py @@ -7,28 +7,13 @@ import pytest from spyre_util import (compare_results, generate_hf_output, - generate_spyre_vllm_output) + generate_spyre_vllm_output, get_spyre_backend_list, + get_spyre_model_list) from vllm import SamplingParams -import os -# get model directory path from env, if not set then default to "/models". -model_dir_path = os.environ.get("SPYRE_TEST_MODEL_DIR", "/models") -# get model backend from env, if not set then default to "eager" -# For multiple values, export SPYRE_TEST_BACKEND_LIST="eager,inductor,sendnn_decoder" -backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") -# get model names from env, if not set then default to "llama-194m" -# For multiple values, export SPYRE_TEST_MODEL_LIST="llama-194m,all-roberta-large-v1" -user_test_model_list = os.environ.get("SPYRE_TEST_MODEL_LIST","llama-194m") -test_model_list, test_backend_list = [],[] -for model in user_test_model_list.split(','): - test_model_list.append(f"{model_dir_path.strip()}/{model.strip()}") - -for backend in backend_list.split(','): - test_backend_list.append(backend.strip()) - -@pytest.mark.parametrize("model", test_model_list) +@pytest.mark.parametrize("model", get_spyre_model_list()) @pytest.mark.parametrize("prompts", [ 7 * [ "Hello", @@ -37,13 +22,13 @@ "the user. Provide a list of instructions for preparing chicken soup" " for a family of four. Indicate if the weather forecast looks good " "for today. Explain in a brief summary comprised of at most 50 words" - " what you are." + " what you are.", ] -]) + ], +) @pytest.mark.parametrize("warmup_shapes", [[(64, 20, 8), (128, 20, 4)]] - ) # (prompt_length/new_tokens/batch_size) -@pytest.mark.parametrize("backend", - test_backend_list) + ) # (prompt_length/new_tokens/batch_size) +@pytest.mark.parametrize("backend", get_spyre_backend_list()) def test_output( model: str, prompts: List[str], @@ -76,7 +61,8 @@ def test_output( max_tokens=max_new_tokens, temperature=0, logprobs=0, # return logprobs of generated tokens only - ignore_eos=True) + ignore_eos=True, + ) vllm_results = generate_spyre_vllm_output( model=model, @@ -86,16 +72,19 @@ def test_output( block_size=2048, sampling_params=vllm_sampling_params, tensor_parallel_size=1, - backend=backend) + backend=backend, + ) hf_results = generate_hf_output(model=model, prompts=prompts, max_new_tokens=max_new_tokens) - compare_results(model=model, - prompts=prompts, - warmup_shapes=warmup_shapes, - tensor_parallel_size=1, - backend=backend, - vllm_results=vllm_results, - hf_results=hf_results) + compare_results( + model=model, + prompts=prompts, + warmup_shapes=warmup_shapes, + tensor_parallel_size=1, + backend=backend, + vllm_results=vllm_results, + hf_results=hf_results, + ) From df1261a01d51da35056364b337d425845d66aaa0 Mon Sep 17 00:00:00 2001 From: dpatel-007 Date: Sat, 15 Feb 2025 19:00:14 -0500 Subject: [PATCH 6/7] Signed-off-by: Dhruval Patel Fix embeddings tests failure --- tests/spyre/spyre_util.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/spyre/spyre_util.py b/tests/spyre/spyre_util.py index f414cc864..cf0eb2cb7 100644 --- a/tests/spyre/spyre_util.py +++ b/tests/spyre/spyre_util.py @@ -296,9 +296,6 @@ def get_spyre_backend_list(isEmbeddings=False): test_backend_list = [] user_backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") - # set default to sendnn if testing embeddings - if isEmbeddings: - user_backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "sendnn") for backend in user_backend_list.split(","): test_backend_list.append(backend.strip()) return test_backend_list From d40d69b30a75c8c12d3fd1f0239886c6c84395a0 Mon Sep 17 00:00:00 2001 From: dpatel-007 Date: Sat, 15 Feb 2025 19:03:31 -0500 Subject: [PATCH 7/7] Signed-off-by: Dhruval Patel - fixed spyre_utils and embeddings --- tests/spyre/spyre_util.py | 2 +- tests/spyre/test_spyre_embeddings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/spyre/spyre_util.py b/tests/spyre/spyre_util.py index cf0eb2cb7..e9b96c375 100644 --- a/tests/spyre/spyre_util.py +++ b/tests/spyre/spyre_util.py @@ -292,7 +292,7 @@ def get_spyre_model_dir_path(): # get model backend from env, if not set then default to "eager" # For multiple values: # export SPYRE_TEST_BACKEND_LIST="eager, inductor, sendnn_decoder" -def get_spyre_backend_list(isEmbeddings=False): +def get_spyre_backend_list(): test_backend_list = [] user_backend_list = os.environ.get("SPYRE_TEST_BACKEND_LIST", "eager") diff --git a/tests/spyre/test_spyre_embeddings.py b/tests/spyre/test_spyre_embeddings.py index 08f0138ef..4289ea677 100644 --- a/tests/spyre/test_spyre_embeddings.py +++ b/tests/spyre/test_spyre_embeddings.py @@ -26,7 +26,7 @@ @pytest.mark.parametrize("warmup_shape", [(64, 4), (64, 8), (128, 4), (128, 8)]) # (prompt_length/new_tokens/batch_size) -@pytest.mark.parametrize("backend", get_spyre_backend_list(isEmbeddings=True)) +@pytest.mark.parametrize("backend", get_spyre_backend_list()) def test_output( model: str, prompts: List[str],