fix all broken tests

DaltheCow · DaltheCow · commit 79c03cefefa9 · 2024-07-23T18:06:46.000-04:00
diff --git a/src/guidellm/executor/__init__.py b/src/guidellm/executor/__init__.py
@@ -1,6 +1,7 @@
 from .executor import Executor
 from .profile_generator import (
     rate_type_to_load_gen_mode,
+    rate_type_to_profile_mode,
     Profile,
     ProfileGenerationMode,
     ProfileGenerator,
@@ -10,6 +11,7 @@
 
 __all__ = [
     "rate_type_to_load_gen_mode",
+    "rate_type_to_profile_mode",
     "Executor",
     "ProfileGenerationMode",
     "Profile",
diff --git a/src/guidellm/executor/executor.py b/src/guidellm/executor/executor.py
@@ -20,20 +20,13 @@ def __init__(
         self,
         backend: Backend,
         request_generator: RequestGenerator,
-        rate_type: str = "sweep",
+        profile_mode: ProfileGenerationMode = ProfileGenerationMode.SWEEP,
         profile_args: Optional[Dict[str, Any]] = None,
         max_requests: Optional[int] = None,
         max_duration: Optional[float] = None,
     ):
         self.request_generator = request_generator
         self.backend = backend
-        profile_mode = "sweep"
-        if rate_type in {"synchronous", "constant", "poisson"}:
-            profile_mode = ProfileGenerationMode.FIXED_RATE
-        elif rate_type == "sweep":
-            profile_mode = ProfileGenerationMode.SWEEP
-        else:
-            raise ValueError("invalid rate type provided")
         self.profile_generator: ProfileGenerator = ProfileGenerator.create(
             profile_mode, **(profile_args or {})
         )
diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py
@@ -22,10 +22,17 @@
     "poisson": LoadGenerationMode.POISSON
 }
 
+
 class ProfileGenerationMode(Enum):
     FIXED_RATE = "fixed_rate"
     SWEEP = "sweep"
 
+rate_type_to_profile_mode = {
+    "synchronous": ProfileGenerationMode.FIXED_RATE,
+    "constant": ProfileGenerationMode.FIXED_RATE,
+    "poisson": ProfileGenerationMode.FIXED_RATE,
+    "sweep": ProfileGenerationMode.SWEEP,
+}
 
 @dataclass
 class Profile:
diff --git a/src/guidellm/main.py b/src/guidellm/main.py
@@ -2,7 +2,7 @@
 
 from guidellm.backend import Backend
 from guidellm.core import TextGenerationBenchmarkReport
-from guidellm.executor import Executor, rate_type_to_load_gen_mode
+from guidellm.executor import Executor, rate_type_to_load_gen_mode, rate_type_to_profile_mode
 from guidellm.request import (
     EmulatedRequestGenerator,
     FileRequestGenerator,
@@ -108,12 +108,14 @@ def main(
     else:
         raise ValueError(f"Unknown data type: {data_type}")
     
+    profile_mode = rate_type_to_profile_mode(rate_type)
     load_gen_mode = rate_type_to_load_gen_mode.get(rate_type, None)
+    
     # Create executor
     executor = Executor(
         request_generator=request_generator,
         backend=backend,
-        rate_type=rate_type,
+        profile_mode=profile_mode,
         profile_args={"load_gen_mode": load_gen_mode, "rates": rate},
         max_requests=num_requests,
         max_duration=num_seconds,
diff --git a/tests/integration/executor/test_report_generation.py b/tests/integration/executor/test_report_generation.py
@@ -18,8 +18,7 @@ def test_executor_openai_single_report_generation_sync_mode(
     )
     profile_generation_mode = ProfileGenerationMode.FIXED_RATE
     profile_generator_kwargs = {
-        "rate_type": LoadGenerationMode.SYNCHRONOUS,
-        "rate": [1.0],
+        "load_gen_mode": LoadGenerationMode.SYNCHRONOUS,
     }
 
     executor = Executor(
@@ -57,8 +56,8 @@ def test_executor_openai_single_report_generation_constant_mode_infinite(
     )
     profile_generation_mode = ProfileGenerationMode.FIXED_RATE
     profile_generator_kwargs = {
-        "rate_type": LoadGenerationMode.CONSTANT,
-        "rate": [1.0],
+        "load_gen_mode": LoadGenerationMode.CONSTANT,
+        "rates": [1.0],
     }
 
     executor = Executor(
@@ -90,8 +89,8 @@ def test_executor_openai_single_report_generation_constant_mode_limited(
     )
     profile_generation_mode = ProfileGenerationMode.FIXED_RATE
     profile_generator_kwargs = {
-        "rate_type": LoadGenerationMode.CONSTANT,
-        "rate": [1.0],
+        "load_gen_mode": LoadGenerationMode.CONSTANT,
+        "rates": [1.0],
     }
 
     executor = Executor(
@@ -126,8 +125,8 @@ def test_executor_openai_single_report_generation_constant_mode_failed(
     )
     profile_generation_mode = ProfileGenerationMode.FIXED_RATE
     profile_generator_kwargs = {
-        "rate_type": LoadGenerationMode.CONSTANT,
-        "rate": [1.0],
+        "load_gen_mode": LoadGenerationMode.CONSTANT,
+        "rates": [1.0],
     }
 
     executor = Executor(
@@ -155,8 +154,8 @@ def test_executor_openai_single_report_generation_constant_mode_cancelled_report
     )
     profile_generation_mode = ProfileGenerationMode.FIXED_RATE
     profile_generator_kwargs = {
-        "rate_type": LoadGenerationMode.CONSTANT,
-        "rate": [1.0],
+        "load_gen_mode": LoadGenerationMode.CONSTANT,
+        "rates": [1.0],
     }
 
     executor = Executor(
diff --git a/tests/unit/executor/test_executor.py b/tests/unit/executor/test_executor.py
@@ -3,17 +3,18 @@
 from guidellm.backend.base import Backend
 from guidellm.executor import Executor
 from guidellm.executor import Profile, ProfileGenerator
+from guidellm.executor.profile_generator import ProfileGenerationMode
 from guidellm.request.base import RequestGenerator
 from guidellm.scheduler import LoadGenerationMode
 
 def test_executor_creation():
     mock_request_generator = MagicMock(spec=RequestGenerator)
     mock_backend = MagicMock(spec=Backend)
-    rate_type = "sweep"
+    profile_mode = ProfileGenerationMode.SWEEP
     profile_args = None
     max_requests = None,
     max_duration = None,
-    executor = Executor(mock_backend, mock_request_generator, rate_type, profile_args, max_requests, max_duration);
+    executor = Executor(mock_backend, mock_request_generator, profile_mode, profile_args, max_requests, max_duration);
     assert executor.request_generator == mock_request_generator
     assert executor.backend == mock_backend
     assert executor.max_requests == max_requests
@@ -47,8 +48,8 @@ def test_executor_run(mock_request_generator, mock_backend, mock_scheduler):
         executor = Executor(
             request_generator=mock_request_generator,
             backend=mock_backend,
-            rate_type="constant",
-            profile_args={"rate_type": "constant", "rate": [1.0, 2.0]},
+            profile_mode=ProfileGenerationMode.FIXED_RATE,
+            profile_args={"load_gen_mode": LoadGenerationMode.CONSTANT, "rates": [1.0, 2.0]},
             max_requests=10,
             max_duration=100
         )
diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py
@@ -65,7 +65,7 @@ def test_rate_doubles():
   mock_report = MagicMock(spec=TextGenerationBenchmarkReport)
   mock_benchmark = MagicMock(spec=TextGenerationBenchmark)
   mock_benchmark.overloaded = False
-  mock_benchmark.args_rate = 2.0
+  mock_benchmark.rate = 2.0
   mock_benchmark.request_rate = 2.0
   benchmarks = [
         mock_benchmark
@@ -81,11 +81,11 @@ def test_max_found():
   mock_report = MagicMock(spec=TextGenerationBenchmarkReport)
   mock_benchmark = MagicMock(spec=TextGenerationBenchmark)
   mock_benchmark.overloaded = False
-  mock_benchmark.args_rate = 2.0
+  mock_benchmark.rate = 2.0
   mock_benchmark.request_rate = 2.0
   mock_overloaded_benchmark = MagicMock(spec=TextGenerationBenchmark)
   mock_overloaded_benchmark.overloaded = True
-  mock_overloaded_benchmark.args_rate = 4.0
+  mock_overloaded_benchmark.rate = 4.0
   mock_overloaded_benchmark.request_rate = 4.0
   benchmarks = [
         mock_benchmark,
@@ -104,11 +104,11 @@ def test_pending_rates():
   mock_report = MagicMock(spec=TextGenerationBenchmarkReport)
   mock_benchmark = MagicMock(spec=TextGenerationBenchmark)
   mock_benchmark.overloaded = False
-  mock_benchmark.args_rate = 2.0
+  mock_benchmark.rate = 2.0
   mock_benchmark.request_rate = 2.0
   mock_overloaded_benchmark = MagicMock(spec=TextGenerationBenchmark)
   mock_overloaded_benchmark.overloaded = True
-  mock_overloaded_benchmark.args_rate = 8.0
+  mock_overloaded_benchmark.rate = 8.0
   mock_overloaded_benchmark.request_rate = 8.0
   benchmarks = [
         mock_benchmark,
diff --git a/tests/unit/executor/test_single_profile_generation_mode.py b/tests/unit/executor/test_single_profile_generation_mode.py
@@ -24,12 +24,15 @@ def test_executor_single_profile_generator_benchmark_report(
     request_genrator = dummy.services.TestRequestGenerator(
         tokenizer="bert-base-uncased"
     )
-    profile_generator_kwargs = {"rate_type": load_gen_mode, "rate": 1.0}
+    rates = [1.0]
+    if (load_gen_mode == LoadGenerationMode.SYNCHRONOUS):
+        rates = None
+    profile_generator_kwargs = {"load_gen_mode": load_gen_mode, "rates": rates}
 
     executor = Executor(
         backend=openai_backend_factory(),
         request_generator=request_genrator,
-        profile_mode=ProfileGenerationMode.SINGLE,
+        profile_mode=ProfileGenerationMode.FIXED_RATE,
         profile_args=profile_generator_kwargs,
         max_requests=1,
         max_duration=None,

Original file line number	Diff line number	Diff line change
`@@ -18,8 +18,7 @@ def test_executor_openai_single_report_generation_sync_mode(`
`18`	`18`	`)`
`19`	`19`	`profile_generation_mode = ProfileGenerationMode.FIXED_RATE`
`20`	`20`	`profile_generator_kwargs = {`
`21`		`- "rate_type": LoadGenerationMode.SYNCHRONOUS,`
`22`		`- "rate": [1.0],`
	`21`	`+ "load_gen_mode": LoadGenerationMode.SYNCHRONOUS,`
`23`	`22`	`}`
`24`	`23`
`25`	`24`	`executor = Executor(`
`@@ -57,8 +56,8 @@ def test_executor_openai_single_report_generation_constant_mode_infinite(`
`57`	`56`	`)`
`58`	`57`	`profile_generation_mode = ProfileGenerationMode.FIXED_RATE`
`59`	`58`	`profile_generator_kwargs = {`
`60`		`- "rate_type": LoadGenerationMode.CONSTANT,`
`61`		`- "rate": [1.0],`
	`59`	`+ "load_gen_mode": LoadGenerationMode.CONSTANT,`
	`60`	`+ "rates": [1.0],`
`62`	`61`	`}`
`63`	`62`
`64`	`63`	`executor = Executor(`
`@@ -90,8 +89,8 @@ def test_executor_openai_single_report_generation_constant_mode_limited(`
`90`	`89`	`)`
`91`	`90`	`profile_generation_mode = ProfileGenerationMode.FIXED_RATE`
`92`	`91`	`profile_generator_kwargs = {`
`93`		`- "rate_type": LoadGenerationMode.CONSTANT,`
`94`		`- "rate": [1.0],`
	`92`	`+ "load_gen_mode": LoadGenerationMode.CONSTANT,`
	`93`	`+ "rates": [1.0],`
`95`	`94`	`}`
`96`	`95`
`97`	`96`	`executor = Executor(`
`@@ -126,8 +125,8 @@ def test_executor_openai_single_report_generation_constant_mode_failed(`
`126`	`125`	`)`
`127`	`126`	`profile_generation_mode = ProfileGenerationMode.FIXED_RATE`
`128`	`127`	`profile_generator_kwargs = {`
`129`		`- "rate_type": LoadGenerationMode.CONSTANT,`
`130`		`- "rate": [1.0],`
	`128`	`+ "load_gen_mode": LoadGenerationMode.CONSTANT,`
	`129`	`+ "rates": [1.0],`
`131`	`130`	`}`
`132`	`131`
`133`	`132`	`executor = Executor(`
`@@ -155,8 +154,8 @@ def test_executor_openai_single_report_generation_constant_mode_cancelled_report`
`155`	`154`	`)`
`156`	`155`	`profile_generation_mode = ProfileGenerationMode.FIXED_RATE`
`157`	`156`	`profile_generator_kwargs = {`
`158`		`- "rate_type": LoadGenerationMode.CONSTANT,`
`159`		`- "rate": [1.0],`
	`157`	`+ "load_gen_mode": LoadGenerationMode.CONSTANT,`
	`158`	`+ "rates": [1.0],`
`160`	`159`	`}`
`161`	`160`
`162`	`161`	`executor = Executor(`