Fix bugs when using fixed_schedule mode

Fix test errors, make timestamp optional field and set default value None to avoid adding it to inputs.json when not using fixed schedule mode Fix codeql Add new line in README Fix pre-commit Revert clang format to what was intended for ubuntu 24.04 Fix pre-commit error
triton-inference-server · Feb 13, 2025 · ab6aaec · ab6aaec
1 parent 069e403
commit ab6aaec
Show file tree

Hide file tree

Showing 33 changed files with 172 additions and 150 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -25,63 +25,63 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 repos:
-- repo: https://github.com/timothycrosley/isort
-  rev: 5.12.0
-  hooks:
-  - id: isort
-    additional_dependencies: [toml]
-- repo: https://github.com/psf/black
-  rev: 24.4.0
-  hooks:
-  - id: black
-    types_or: [python, cython]
-- repo: https://github.com/PyCQA/flake8
-  rev: 5.0.4
-  hooks:
-  - id: flake8
-    args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501]
-    types_or: [python, cython]
-- repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: v18.1.3
-  hooks:
-  - id: clang-format
-    types_or: [c, c++, cuda, proto, textproto, java]
-    args: ["-fallback-style=none", "-style=file", "-i"]
-- repo: https://github.com/codespell-project/codespell
-  rev: v2.2.4
-  hooks:
-  - id: codespell
-    additional_dependencies: [tomli]
-    args: ["--toml", "pyproject.toml"]
-    exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$|.*shakespeare.txt)
-# More details about these pre-commit hooks here:
-# https://pre-commit.com/hooks.html
-- repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
-  hooks:
-  - id: check-case-conflict
-  - id: check-executables-have-shebangs
-  - id: check-merge-conflict
-  - id: check-json
-  - id: check-toml
-  - id: check-yaml
-  - id: check-shebang-scripts-are-executable
-  - id: end-of-file-fixer
-    types_or: [c, c++, cuda, proto, textproto, java, python]
-  - id: mixed-line-ending
-  - id: requirements-txt-fixer
-  - id: trailing-whitespace
+  - repo: https://github.com/timothycrosley/isort
+    rev: 5.12.0
+    hooks:
+    - id: isort
+      additional_dependencies: [toml]
+  - repo: https://github.com/psf/black
+    rev: 24.4.0
+    hooks:
+    - id: black
+      types_or: [python, cython]
+  - repo: https://github.com/PyCQA/flake8
+    rev: 5.0.4
+    hooks:
+    - id: flake8
+      args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501]
+      types_or: [python, cython]
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: v18.1.3
+    hooks:
+    - id: clang-format
+      types_or: [c, c++, cuda, proto, textproto, java]
+      args: ["-fallback-style=none", "-style=file", "-i"]
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.4
+    hooks:
+    - id: codespell
+      additional_dependencies: [tomli]
+      args: ["--toml", "pyproject.toml"]
+      exclude: (?x)^(.*stemmer.*|.*stop_words.*|^CHANGELOG.md$|.*shakespeare.txt)
+  # More details about these pre-commit hooks here:
+  # https://pre-commit.com/hooks.html
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    - id: check-case-conflict
+    - id: check-executables-have-shebangs
+    - id: check-merge-conflict
+    - id: check-json
+    - id: check-toml
+    - id: check-yaml
+    - id: check-shebang-scripts-are-executable
+    - id: end-of-file-fixer
+      types_or: [c, c++, cuda, proto, textproto, java, python]
+    - id: mixed-line-ending
+    - id: requirements-txt-fixer
+    - id: trailing-whitespace
 
-- repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.9.0
-  hooks:
-  - id: mypy
-    types_or: [python, cython]
-    additional_dependencies: [tokenize-rt==3.2.0, types-requests]
-- repo: local
-  hooks:
-  - id: add-license
-    name: Add License
-    entry: python tools/add_copyright.py
-    language: python
-    require_serial: true
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.9.0
+    hooks:
+    - id: mypy
+      types_or: [python, cython]
+      additional_dependencies: [tokenize-rt==3.2.0, types-requests]
+  - repo: local
+    hooks:
+    - id: add-license
+      name: Add License
+      entry: python tools/add_copyright.py
+      language: python
+      require_serial: true
diff --git a/genai-perf/README.md b/genai-perf/README.md
@@ -481,6 +481,7 @@ Add a custom header to the requests. Headers must be specified as
 (default: `None`)
 
 ##### `--input-file <path>`
+
 The input file or directory for profiling. Each line should be a JSON object
 with a `text` or `image` field in JSONL format. Example:
 `{"text": "Your prompt here"}`. To use synthetic files, prefix with

diff --git a/genai-perf/docs/customizable_frontends.md b/genai-perf/docs/customizable_frontends.md
@@ -1,5 +1,5 @@
 <!--
-Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions

diff --git a/genai-perf/genai_perf/inputs/converters/base_converter.py b/genai-perf/genai_perf/inputs/converters/base_converter.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -89,7 +89,7 @@ def _finalize_payload(
             record["payload"] = [payload]
         else:
             record.update(payload)
-        if row.timestamp:
+        if row.timestamp is not None:
             record["timestamp"] = [row.timestamp]
 
         return record
diff --git a/genai-perf/genai_perf/inputs/converters/image_retrieval_converter.py b/genai-perf/genai_perf/inputs/converters/image_retrieval_converter.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

diff --git a/genai-perf/genai_perf/inputs/converters/nvclip_converter.py b/genai-perf/genai_perf/inputs/converters/nvclip_converter.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

diff --git a/genai-perf/genai_perf/inputs/converters/openai_chat_completions_converter.py b/genai-perf/genai_perf/inputs/converters/openai_chat_completions_converter.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

diff --git a/genai-perf/genai_perf/inputs/converters/openai_completions_converter.py b/genai-perf/genai_perf/inputs/converters/openai_completions_converter.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

diff --git a/genai-perf/genai_perf/inputs/converters/openai_embeddings_converter.py b/genai-perf/genai_perf/inputs/converters/openai_embeddings_converter.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

diff --git a/genai-perf/genai_perf/inputs/converters/rankings_converter.py b/genai-perf/genai_perf/inputs/converters/rankings_converter.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

diff --git a/genai-perf/genai_perf/inputs/converters/tensorrtllm_converter.py b/genai-perf/genai_perf/inputs/converters/tensorrtllm_converter.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

diff --git a/genai-perf/genai_perf/inputs/converters/triton_generate_converter.py b/genai-perf/genai_perf/inputs/converters/triton_generate_converter.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

diff --git a/genai-perf/genai_perf/inputs/converters/vllm_converter.py b/genai-perf/genai_perf/inputs/converters/vllm_converter.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

diff --git a/genai-perf/genai_perf/inputs/retrievers/base_file_input_retriever.py b/genai-perf/genai_perf/inputs/retrievers/base_file_input_retriever.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -56,7 +56,7 @@ def _verify_file(self, filename: Path) -> None:
 
     def _get_content_from_input_file(self, filename: Path) -> Union[
         Tuple[List[str], List[str]],
-        Tuple[List[str], List[str], List[Dict[Any, Any]]],
+        Tuple[List[str], List[int], List[Dict[Any, Any]]],
     ]:
         """
         Reads the content from a JSONL file and returns lists of each content type.

diff --git a/genai-perf/genai_perf/inputs/retrievers/file_input_retriever.py b/genai-perf/genai_perf/inputs/retrievers/file_input_retriever.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

diff --git a/genai-perf/genai_perf/inputs/retrievers/generic_dataset.py b/genai-perf/genai_perf/inputs/retrievers/generic_dataset.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -25,14 +25,14 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, TypeAlias, Union
+from typing import Any, Dict, List, Optional, TypeAlias, Union
 
 Filename: TypeAlias = str
 TextData: TypeAlias = List[str]
 ImageData: TypeAlias = List[str]
 InputData: TypeAlias = Union[TextData, ImageData]
 OptionalData: TypeAlias = Dict[str, Any]
-Timestamp: TypeAlias = str
+Timestamp: TypeAlias = int
 DataRowDict: TypeAlias = Dict[str, Union[InputData, Timestamp, OptionalData]]
 GenericDatasetDict: TypeAlias = Dict[Filename, List[DataRowDict]]
 
@@ -41,7 +41,7 @@
 class DataRow:
     texts: List[str] = field(default_factory=list)
     images: List[str] = field(default_factory=list)
-    timestamp: str = ""
+    timestamp: Optional[int] = None
     optional_data: Dict[str, Any] = field(default_factory=dict)
 
     def to_dict(self) -> DataRowDict:

diff --git a/genai-perf/genai_perf/inputs/retrievers/input_retriever_factory.py b/genai-perf/genai_perf/inputs/retrievers/input_retriever_factory.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

diff --git a/genai-perf/genai_perf/inputs/retrievers/payload_input_retriever.py b/genai-perf/genai_perf/inputs/retrievers/payload_input_retriever.py
@@ -89,7 +89,7 @@ def _get_input_dataset_from_file(self, filename: Path) -> FileData:
 
     def _get_content_from_input_file(
         self, filename: Path
-    ) -> Tuple[List[str], List[str], List[Dict[Any, Any]]]:
+    ) -> Tuple[List[str], List[int], List[Dict[Any, Any]]]:
         """
         Reads the content from a JSONL file and returns lists of each content type.
 
@@ -127,27 +127,49 @@ def _get_content_from_input_file(
                         )
                     prompt = prompt if prompt else prompt_alt
                     prompts.append(prompt.strip() if prompt else prompt)
-                    timestamp = data.get("timestamp", "")
+                    timestamp = self._get_valid_timestamp(data)
                     timestamps.append(timestamp)
                     optional_data = self._check_for_optional_data(data)
                     optional_datas.append(optional_data)
         return prompts, timestamps, optional_datas
 
+    def _get_valid_timestamp(self, data: Dict[str, Any]) -> int:
+        """
+        Retrieves and validates timestamp from input data
+        """
+        timestamp = data.get("timestamp")
+        if timestamp is None:
+            raise ValueError("Each data entry must have a 'timestamp' field.")
+        try:
+            timestamp = int(timestamp)
+        except ValueError:
+            raise ValueError("Invalid timestamp value.")
+
+        return timestamp
+
     def _check_for_optional_data(self, data: Dict[str, Any]) -> Dict[Any, Any]:
         """
         Checks if there is any optional data in the file to pass in the payload.
         """
         optional_data = {
             k: v
             for k, v in data.items()
-            if k not in ["text", "text_input", "timestamp", "hash_ids"]
+            if k
+            not in [
+                "text",
+                "text_input",
+                "timestamp",
+                "hash_ids",
+                "input_length",
+                "output_length",
+            ]
         }
         return optional_data
 
     def _convert_content_to_data_file(
         self,
         prompts: List[str],
-        timestamps: List[str],
+        timestamps: List[int],
         optional_datas: List[Dict[Any, Any]] = [{}],
     ) -> FileData:
         """
@@ -157,7 +179,7 @@ def _convert_content_to_data_file(
         ----------
         prompts : List[str]
             The list of prompts to convert.
-        timestamps: str
+        timestamps: int
             The timestamp at which the request should be sent.
         optional_data : Dict
             The optional data included in every payload.

diff --git a/genai-perf/genai_perf/subcommand/profile.py b/genai-perf/genai_perf/subcommand/profile.py
@@ -29,6 +29,7 @@
 
 from genai_perf.exceptions import GenAIPerfException
 from genai_perf.export_data.output_reporter import OutputReporter
+from genai_perf.inputs import input_constants as ic
 from genai_perf.metrics.telemetry_statistics import TelemetryStatistics
 from genai_perf.plots.plot_config_parser import PlotConfigParser
 from genai_perf.plots.plot_manager import PlotManager
@@ -81,6 +82,9 @@ def _report_output(
     elif args.request_rate:
         infer_mode = "request_rate"
         load_level = f"{args.request_rate}"
+    elif args.prompt_source == ic.PromptSource.PAYLOAD:
+        infer_mode = "request_rate"
+        load_level = "1.0"
     else:
         raise GenAIPerfException("No valid infer mode specified")
 

diff --git a/genai-perf/genai_perf/wrapper.py b/genai-perf/genai_perf/wrapper.py
@@ -24,7 +24,6 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import json
 from argparse import Namespace
 from typing import List, Optional