Add Support for Templating to GAP (#283)

* Adding support for templating to GAP

* Potential fix for code scanning alert no. 159: Unused import

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* Fixing isort issue created by autofix

* Refactoring template creation

* Refactoring parser

* Adding missing verbose template comment

* Reformatting sweep template comment

---------

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

Author: nv-braf

genai-perf/genai_perf/checkpoint/checkpoint.py

@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -47,6 +47,8 @@ def __post_init__(self):
     # Read/Write Methods
     ###########################################################################
     def create_checkpoint_object(self) -> None:
+        os.makedirs(self.config.output.checkpoint_directory, exist_ok=True)
+
         state_dict = {"Results": self.results.create_checkpoint_object()}
 
         checkpoint_file_path = self._create_checkpoint_file_path()

genai-perf/genai_perf/config/input/base_config.py

@@ -15,6 +15,8 @@
 import copy
 from enum import Enum
 from pathlib import PosixPath
+from textwrap import indent
+from typing import Any, Dict
 
 from genai_perf.config.input.config_field import ConfigField
 
@@ -40,13 +42,16 @@ def __init__(self):
         # This exists just to make looking up values when debugging easier
         self._values = {}
 
-    def get_field(self, name):
+    ###########################################################################
+    # Top-Level Methods
+    ###########################################################################
+    def get_field(self, name) -> ConfigField:
         if name not in self._fields:
             raise ValueError(f"{name} not found in ConfigFields")
 
         return self._fields[name]
 
-    def to_json_dict(self):
+    def to_json_dict(self) -> Dict[str, Any]:
         config_dict = {}
         for key, value in self._values.items():
             if isinstance(value, BaseConfig):
@@ -56,7 +61,7 @@ def to_json_dict(self):
 
         return config_dict
 
-    def _get_legal_json_value(self, value):
+    def _get_legal_json_value(self, value: Any) -> Any:
         if isinstance(value, Enum):
             return value.name.lower()
         elif isinstance(value, PosixPath):
@@ -82,6 +87,75 @@ def _get_legal_json_value(self, value):
         else:
             raise ValueError(f"Value {value} is not a legal JSON value")
 
+    ###########################################################################
+    # Template Creation Methods
+    ###########################################################################
+    def create_template(self, header: str, level: int = 1, verbose=False) -> str:
+        indention = "  " * level
+
+        template = self._add_header_to_template(header, indention)
+        template += self._add_fields_to_template(indention, verbose)
+        template += "\n"
+        template += self._add_children_to_template(level, verbose)
+
+        return template
+
+    def _add_header_to_template(self, header: str, indention: str) -> str:
+        template = ""
+        if header:
+            template = indent(f"{header}:\n", indention)
+        return template
+
+    def _add_fields_to_template(self, indention: str, verbose: bool) -> str:
+        template = ""
+        for name, field in self._fields.items():
+            template_comment = self._get_template_comment(field, verbose)
+            template += self._create_template_from_comment(template_comment, indention)
+            template += self._add_field_to_template(field, name, indention)
+
+            if verbose and field.verbose_template_comment:
+                template += "\n"
+
+        return template
+
+    def _add_children_to_template(self, level: int, verbose: bool) -> str:
+        template = ""
+        for name, child in self._children.items():
+            template += child.create_template(
+                header=name, level=level + 1, verbose=verbose
+            )
+
+        return template
+
+    def _get_template_comment(self, field: ConfigField, verbose: bool) -> str:
+        if verbose and field.verbose_template_comment:
+            return field.verbose_template_comment
+        else:
+            return field.template_comment if field.template_comment else ""
+
+    def _create_template_from_comment(self, comment: str, indention: str) -> str:
+        template = ""
+        if comment:
+            comment_lines = comment.split("\n")
+            for comment_line in comment_lines:
+                template += indent(f"  # {comment_line}\n", indention)
+
+        return template
+
+    def _add_field_to_template(
+        self, field: ConfigField, name: str, indention: str
+    ) -> str:
+        template = ""
+        if field.add_to_template:
+            template = indent(
+                f"  {name}: {self._get_legal_json_value(self.__getattr__(name))}\n",
+                indention,
+            )
+        return template
+
+    ###########################################################################
+    # Dunder Methods
+    ###########################################################################
     def __setattr__(self, name, value):
         # This prevents recursion failure in __init__
         if name == "_fields" or name == "_values" or name == "_children":

genai-perf/genai_perf/config/input/config_analyze.py

@@ -31,11 +31,29 @@ class ConfigAnalyze(BaseConfig):
 
     def __init__(self) -> None:
         super().__init__()
+        # yapf: disable
+        sweep_parameter_template_comment = \
+        (f"Uncomment the lines below to enable the analyze subcommand\n"
+         f"For further details see analyze.md sweep_parameters:\n"
+         f"  concurrency:\n"
+         f"    start: {AnalyzeDefaults.MIN_CONCURRENCY}\n"
+         f"    stop: {AnalyzeDefaults.MAX_CONCURRENCY}")
+        # yapf: enable
+
         self.sweep_parameters: Any = ConfigField(
-            default=AnalyzeDefaults.SWEEP_PARAMETER, choices=all_parameters
+            default=AnalyzeDefaults.SWEEP_PARAMETER,
+            choices=all_parameters,
+            add_to_template=False,
+            template_comment=sweep_parameter_template_comment,
         )
 
+    ###########################################################################
+    # Parsing Methods
+    ###########################################################################
     def parse(self, analyze: Dict[str, Any]) -> None:
+        if not analyze:
+            return
+
         sweep_parameters: Dict[str, Any] = {}
         for sweep_type, range_dict in analyze.items():
             if (

genai-perf/genai_perf/config/input/config_command.py

@@ -38,6 +38,7 @@ class Subcommand(Enum):
     COMPARE = "compare"
     PROFILE = "profile"
     ANALYZE = "analyze"
+    TEMPLATE = "create-template"
 
 
 ConfigRangeOrList: TypeAlias = Optional[Union[Range, List[int]]]
@@ -54,7 +55,10 @@ def __init__(self, user_config: Optional[Dict[str, Any]] = None):
         super().__init__()
 
         self.model_names: Any = ConfigField(
-            default=TopLevelDefaults.MODEL_NAME, required=True
+            default=TopLevelDefaults.MODEL_NAME,
+            required=True,
+            add_to_template=True,
+            verbose_template_comment="The name of the model(s) to benchmark.",
         )
 
         self.analyze = ConfigAnalyze()
@@ -195,6 +199,12 @@ def _process_stimulus(self) -> List[str]:
         else:
             return []
 
+    ###########################################################################
+    # Template Creation Methods
+    ###########################################################################
+    def make_template(self) -> str:
+        return self.create_template(header="", level=0, verbose=self.verbose)
+
     ###########################################################################
     # Utility Methods
     ###########################################################################

genai-perf/genai_perf/config/input/config_defaults.py

@@ -16,6 +16,7 @@
 from dataclasses import dataclass, field
 
 from genai_perf.inputs.input_constants import ModelSelectionStrategy, OutputFormat
+from genai_perf.inputs.retrievers.synthetic_image_generator import ImageFormat
 
 
 def default_field(obj):
@@ -62,8 +63,8 @@ class EndPointDefaults:
     TYPE = ""
     SERVICE_KIND = "triton"
     STREAMING = False
-    SERVER_METRICS_URL = None
-    URL = ""
+    SERVER_METRICS_URL = ["http://localhost:8002/metrics"]
+    URL = "localhost:8001"
 
 
 @dataclass(frozen=True)
@@ -82,7 +83,7 @@ class ImageDefaults:
     WIDTH_STDDEV = 0
     HEIGHT_MEAN = 100
     HEIGHT_STDDEV = 0
-    FORMAT = None
+    FORMAT = ImageFormat.PNG
 
 
 @dataclass(frozen=True)
@@ -113,18 +114,18 @@ class RequestCountDefaults:
 @dataclass(frozen=True)
 class InputDefaults:
     BATCH_SIZE = 1
-    EXTRA = None
-    GOODPUT = None
-    HEADER = None
-    FILE = None
+    EXTRA = ""
+    GOODPUT = ""
+    HEADER = ""
+    FILE = ""
     NUM_DATASET_ENTRIES = 100
     RANDOM_SEED = 0
 
 
 @dataclass(frozen=True)
 class OutputDefaults:
     ARTIFACT_DIRECTORY = "./artifacts"
-    CHECKPOINT_DIRECTORY = "./"
+    CHECKPOINT_DIRECTORY = "./checkpoint"
     PROFILE_EXPORT_FILE = "profile_export.json"
     GENERATE_PLOTS = False
 
@@ -134,3 +135,8 @@ class TokenizerDefaults:
     NAME = "hf-internal-testing/llama-tokenizer"
     REVISION = "main"
     TRUST_REMOTE_CODE = False
+
+
+@dataclass(frozen=True)
+class TemplateDefaults:
+    FILENAME = "genai_perf_config.yaml"

genai-perf/genai_perf/config/input/config_endpoint.py

@@ -32,24 +32,46 @@ def __init__(self) -> None:
         self.model_selection_strategy: Any = ConfigField(
             default=EndPointDefaults.MODEL_SELECTION_STRATEGY,
             choices=ModelSelectionStrategy,
+            verbose_template_comment="When multiple model are specified, this is how a specific model should be assigned to a prompt.\
+            \nround_robin: nth prompt in the list gets assigned to n-mod len(models).\
+            \nrandom: assignment is uniformly random",
         )
         self.backend: Any = ConfigField(
-            default=EndPointDefaults.BACKEND, choices=OutputFormat
+            default=EndPointDefaults.BACKEND,
+            choices=OutputFormat,
+            verbose_template_comment="When using the \"triton\" service-kind, this is the backend of the model.\
+                \nFor the TENSORRT-LLM backend,you currently must set 'exclude_input_in_output' to true\
+                \nin the model config to not echo the input tokens",
+        )
+
+        self.custom: Any = ConfigField(
+            default=EndPointDefaults.CUSTOM,
+            verbose_template_comment="Set a custom endpoint that differs from the OpenAI defaults.",
         )
-        self.custom: Any = ConfigField(default=EndPointDefaults.CUSTOM)
         self.type: Any = ConfigField(
             default=EndPointDefaults.TYPE,
             choices=list(endpoint_type_map.keys()),
+            verbose_template_comment="The type to send requests to on the server.",
         )
         self.service_kind: Any = ConfigField(
             default=EndPointDefaults.SERVICE_KIND,
             choices=["triton", "openai", "tensorrtllm_engine"],
+            verbose_template_comment='The kind of service Perf Analyzer will generate load for.\
+                \nIn order to use "openai", you must specify an api via the "type" field',
+        )
+        self.streaming: Any = ConfigField(
+            default=EndPointDefaults.STREAMING,
+            verbose_template_comment="An option to enable the use of the streaming API.",
         )
-        self.streaming: Any = ConfigField(default=EndPointDefaults.STREAMING)
         self.server_metrics_urls: Any = ConfigField(
-            default=EndPointDefaults.SERVER_METRICS_URL
+            default=EndPointDefaults.SERVER_METRICS_URL,
+            verbose_template_comment='The list of Triton server metrics URLs.\
+                \nThese are used for Telemetry metric reporting with the "triton" service-kind.',
+        )
+        self.url: Any = ConfigField(
+            default=EndPointDefaults.URL,
+            verbose_template_comment="URL of the endpoint to target for benchmarking.",
         )
-        self.url: Any = ConfigField(default=EndPointDefaults.URL)
 
     def parse(self, endpoint: Dict[str, Any]) -> None:
         for key, value in endpoint.items():

genai-perf/genai_perf/config/input/config_field.py

@@ -38,6 +38,7 @@ def __init__(
         required: bool = False,
         add_to_template: bool = True,
         template_comment: Optional[str] = None,
+        verbose_template_comment: Optional[str] = None,
         value: Optional[Any] = None,
         bounds: Optional[Dict[str, Any]] = None,
         choices: Optional[Any] = None,
@@ -46,6 +47,7 @@ def __init__(
         self.required = required
         self.add_to_template = add_to_template
         self.template_comment = template_comment
+        self.verbose_template_comment = verbose_template_comment
         self.bounds = bounds
         self.choices = choices
         self.is_set_by_user = False