Refactors to enable tests for benchmarker package and supporting tools

Author: markurtz

src/guidellm/backend/__init__.py

@@ -14,11 +14,13 @@
     GenerationRequestTimings,
     GenerationResponse,
 )
+from .openai import OpenAIHTTPBackend
 
 __all__ = [
     "Backend",
     "BackendType",
     "GenerationRequest",
     "GenerationRequestTimings",
     "GenerationResponse",
+    "OpenAIHTTPBackend",
 ]

src/guidellm/backend/backend.py

@@ -11,15 +11,18 @@
     BackendType: Literal type defining supported backend implementations.
 """
 
-from typing import Literal, Optional
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import Literal
 
 from guidellm.backend.objects import (
     GenerationRequest,
     GenerationRequestTimings,
     GenerationResponse,
 )
 from guidellm.scheduler import BackendInterface
-from guidellm.utils.registry import RegistryMixin
+from guidellm.utils import RegistryMixin
 
 __all__ = [
     "Backend",
@@ -66,7 +69,7 @@ async def process_startup(self):
     """
 
     @classmethod
-    def create(cls, type_: BackendType, **kwargs) -> "Backend":
+    def create(cls, type_: BackendType, **kwargs) -> Backend:
         """
         Create a backend instance based on the backend type.
 
@@ -78,6 +81,12 @@ def create(cls, type_: BackendType, **kwargs) -> "Backend":
 
         backend = cls.get_registered_object(type_)
 
+        if backend is None:
+            raise ValueError(
+                f"Backend type '{type_}' is not registered. "
+                f"Available types: {list(cls.registry.keys()) if cls.registry else []}"
+            )
+
         return backend(**kwargs)
 
     def __init__(self, type_: BackendType):
@@ -89,16 +98,23 @@ def __init__(self, type_: BackendType):
         self.type_ = type_
 
     @property
-    def processes_limit(self) -> Optional[int]:
+    def processes_limit(self) -> int | None:
         """
         :return: Maximum number of worker processes supported. None if unlimited.
         """
         return None
 
     @property
-    def requests_limit(self) -> Optional[int]:
+    def requests_limit(self) -> int | None:
         """
         :return: Maximum number of concurrent requests supported globally.
             None if unlimited.
         """
         return None
+
+    @abstractmethod
+    async def default_model(self) -> str | None:
+        """
+        :return: The default model name or identifier for generation requests.
+        """
+        ...

src/guidellm/backend/interface.py

@@ -11,8 +11,8 @@
 
 from pydantic import Field
 
-from guidellm.objects.pydantic import StandardBaseModel
 from guidellm.scheduler import MeasuredRequestTimings
+from guidellm.utils import StandardBaseModel
 
 __all__ = [
     "GenerationRequest",

src/guidellm/backend/objects.py

@@ -1,19 +1,29 @@
-from .aggregator import AggregatorT, BenchmarkAggregator, GenerativeBenchmarkAggregator
-from .benchmark import (
+from .aggregator import (
+    Aggregator,
+    CompilableAggregator,
+    GenerativeRequestsAggregator,
+    GenerativeStatsProgressAggregator,
+    SchedulerStatsAggregator,
+    SerializableAggregator,
+)
+from .benchmarker import Benchmarker
+from .entrypoints import benchmark_generative_text, reimport_benchmarks_report
+from .objects import (
     Benchmark,
-    BenchmarkArgs,
     BenchmarkMetrics,
     BenchmarkSchedulerStats,
     BenchmarkT,
     GenerativeBenchmark,
+    GenerativeBenchmarksReport,
     GenerativeMetrics,
     GenerativeRequestStats,
-    GenerativeTextErrorStats,
-    StatusBreakdown,
 )
-from .benchmarker import Benchmarker, BenchmarkerResult, GenerativeBenchmarker
-from .entrypoints import benchmark_generative_text, reimport_benchmarks_report
-from .output import GenerativeBenchmarksConsole, GenerativeBenchmarksReport
+from .output import (
+    GenerativeBenchmarkerConsole,
+    GenerativeBenchmarkerCSV,
+    GenerativeBenchmarkerHTML,
+    GenerativeBenchmarkerOutput,
+)
 from .profile import (
     AsyncProfile,
     ConcurrentProfile,
@@ -22,46 +32,43 @@
     SweepProfile,
     SynchronousProfile,
     ThroughputProfile,
-    create_profile,
 )
 from .progress import (
-    BenchmarkerProgressDisplay,
-    BenchmarkerTaskProgressState,
-    GenerativeTextBenchmarkerProgressDisplay,
-    GenerativeTextBenchmarkerTaskProgressState,
+    BenchmarkerProgress,
+    BenchmarkerProgressGroup,
+    GenerativeConsoleBenchmarkerProgress,
 )
 
 __all__ = [
-    "AggregatorT",
+    "Aggregator",
     "AsyncProfile",
     "Benchmark",
-    "BenchmarkAggregator",
-    "BenchmarkArgs",
     "BenchmarkMetrics",
     "BenchmarkSchedulerStats",
     "BenchmarkT",
     "Benchmarker",
-    "BenchmarkerProgressDisplay",
-    "BenchmarkerResult",
-    "BenchmarkerTaskProgressState",
+    "BenchmarkerProgress",
+    "BenchmarkerProgressGroup",
+    "CompilableAggregator",
     "ConcurrentProfile",
     "GenerativeBenchmark",
-    "GenerativeBenchmarkAggregator",
-    "GenerativeBenchmarker",
-    "GenerativeBenchmarksConsole",
+    "GenerativeBenchmarkerCSV",
+    "GenerativeBenchmarkerConsole",
+    "GenerativeBenchmarkerHTML",
+    "GenerativeBenchmarkerOutput",
     "GenerativeBenchmarksReport",
+    "GenerativeConsoleBenchmarkerProgress",
     "GenerativeMetrics",
     "GenerativeRequestStats",
-    "GenerativeTextBenchmarkerProgressDisplay",
-    "GenerativeTextBenchmarkerTaskProgressState",
-    "GenerativeTextErrorStats",
+    "GenerativeRequestsAggregator",
+    "GenerativeStatsProgressAggregator",
     "Profile",
     "ProfileType",
-    "StatusBreakdown",
+    "SchedulerStatsAggregator",
+    "SerializableAggregator",
     "SweepProfile",
     "SynchronousProfile",
     "ThroughputProfile",
     "benchmark_generative_text",
-    "create_profile",
     "reimport_benchmarks_report",
 ]