Results from GH action on NVIDIA_RTX4090x2

mlcommons · Feb 11, 2025 · 621fdca · 621fdca
1 parent db37fd8
commit 621fdca
Show file tree

Hide file tree

Showing 45 changed files with 1,368 additions and 1,368 deletions.
diff --git a/...RTX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/accuracy.txt b/...RTX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/accuracy.txt
@@ -1,2 +1,2 @@
 
-hash=f140a7bb4620ebc9e683e6611c1efad321151c93af78c6621b0a103f4eb8cb44
+hash=b9c1f4d44fba546a1c278a6dd98967827d1c343011f23814dedbfc7d91ffc5f9
diff --git a/...-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/baseline_accuracy.txt b/...-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/baseline_accuracy.txt
@@ -1,4 +1,4 @@
-{"exact_match": 25.950804162724694, "f1": 28.336484611704563}
+{"exact_match": 25.76158940397351, "f1": 27.99745717265541}
 Reading examples...
 No cached features at 'eval_features.pickle'... converting from examples...
 Creating tokenizer...

diff --git a/...vidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/compliance_accuracy.txt b/...vidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/compliance_accuracy.txt
@@ -1,4 +1,4 @@
-{"exact_match": 25.950804162724694, "f1": 28.336484611704563}
+{"exact_match": 25.75212866603595, "f1": 27.995565025067897}
 Reading examples...
 Loading cached features from 'eval_features.pickle'...
 Loading LoadGen logs...

diff --git a/...idia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/mlperf_log_accuracy.json b/...idia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/mlperf_log_accuracy.json
diff --git a/...pu-TensorRT-default_config/bert-99/offline/TEST01/performance/run_1/mlperf_log_detail.txt b/...pu-TensorRT-default_config/bert-99/offline/TEST01/performance/run_1/mlperf_log_detail.txt
diff --git a/...u-TensorRT-default_config/bert-99/offline/TEST01/performance/run_1/mlperf_log_summary.txt b/...u-TensorRT-default_config/bert-99/offline/TEST01/performance/run_1/mlperf_log_summary.txt
@@ -4,7 +4,7 @@ MLPerf Results Summary
 SUT name : BERT SERVER
 Scenario : Offline
 Mode     : PerformanceOnly
-Samples per second: 3330.98
+Samples per second: 8271.39
 Result is : VALID
   Min duration satisfied : Yes
   Min queries satisfied : Yes
@@ -13,21 +13,21 @@ Result is : VALID
 ================================================
 Additional Stats
 ================================================
-Min latency (ns)                : 1116211759
-Max latency (ns)                : 666841029709
-Mean latency (ns)               : 402978238934
-50.00 percentile latency (ns)   : 428982589124
-90.00 percentile latency (ns)   : 635783152565
-95.00 percentile latency (ns)   : 654262018158
-97.00 percentile latency (ns)   : 660328292889
-99.00 percentile latency (ns)   : 665096268463
-99.90 percentile latency (ns)   : 666746018232
+Min latency (ns)                : 1526539431
+Max latency (ns)                : 665098050935
+Mean latency (ns)               : 403352540223
+50.00 percentile latency (ns)   : 429429672390
+90.00 percentile latency (ns)   : 634301266538
+95.00 percentile latency (ns)   : 652493305166
+97.00 percentile latency (ns)   : 658499859334
+99.00 percentile latency (ns)   : 663297841625
+99.90 percentile latency (ns)   : 664956599197
 
 ================================================
 Test Parameters Used
 ================================================
-samples_per_query : 2221231
-target_qps : 3365.5
+samples_per_query : 5501283
+target_qps : 8335.28
 target_latency (ns): 0
 max_async_queries : 1
 min_duration (ms): 600000

diff --git a/...e/RTX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/verify_accuracy.txt b/...e/RTX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/verify_accuracy.txt
@@ -4,9 +4,9 @@ Reading performance mode results...
 num_acc_log_entries = 10833
 num_acc_log_duplicate_keys = 0
 num_acc_log_data_mismatch = 0
-num_perf_log_entries = 4084
-num_perf_log_qsl_idx_match = 4084
-num_perf_log_data_mismatch = 51
+num_perf_log_entries = 4020
+num_perf_log_qsl_idx_match = 4020
+num_perf_log_data_mismatch = 21
 num_missing_qsl_idxs = 0
 TEST FAIL
 
diff --git a/...TX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/verify_performance.txt b/...TX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/verify_performance.txt
@@ -1,4 +1,4 @@
 Verifying performance.
-reference score = 3332.18
-test score = 3330.98
+reference score = 8252.75
+test score = 8271.39
 TEST PASS
diff --git a/...90x2-nvidia-gpu-TensorRT-default_config/bert-99/singlestream/TEST01/accuracy/accuracy.txt b/...90x2-nvidia-gpu-TensorRT-default_config/bert-99/singlestream/TEST01/accuracy/accuracy.txt
@@ -1,2 +1,2 @@
 
-hash=e07d27d2d42fbf5d05271577e09aff1a3ffde558fb836b04be3d64a30ad109a6
+hash=d589db7eb874fc4c9753153301ee3b630928f4048e1ac13aadca74a742750002
diff --git a/...gpu-TensorRT-default_config/bert-99/singlestream/TEST01/accuracy/mlperf_log_accuracy.json b/...gpu-TensorRT-default_config/bert-99/singlestream/TEST01/accuracy/mlperf_log_accuracy.json
diff --git a/...nsorRT-default_config/bert-99/singlestream/TEST01/performance/run_1/mlperf_log_detail.txt b/...nsorRT-default_config/bert-99/singlestream/TEST01/performance/run_1/mlperf_log_detail.txt
diff --git a/...sorRT-default_config/bert-99/singlestream/TEST01/performance/run_1/mlperf_log_summary.txt b/...sorRT-default_config/bert-99/singlestream/TEST01/performance/run_1/mlperf_log_summary.txt
@@ -4,38 +4,38 @@ MLPerf Results Summary
 SUT name : BERT SERVER
 Scenario : SingleStream
 Mode     : PerformanceOnly
-90th percentile latency (ns) : 2173312
+90th percentile latency (ns) : 1029191
 Result is : VALID
   Min duration satisfied : Yes
   Min queries satisfied : Yes
   Early stopping satisfied: Yes
 Early Stopping Result:
- * Processed at least 64 queries (391435).
- * Would discard 38705 highest latency queries.
- * Early stopping 90th percentile estimate: 2174469
- * Early stopping 99th percentile estimate: 2637128
+ * Processed at least 64 queries (634753).
+ * Would discard 62918 highest latency queries.
+ * Early stopping 90th percentile estimate: 1029636
+ * Early stopping 99th percentile estimate: 1209114
 
 ================================================
 Additional Stats
 ================================================
-QPS w/ loadgen overhead         : 652.39
-QPS w/o loadgen overhead        : 654.77
+QPS w/ loadgen overhead         : 1057.92
+QPS w/o loadgen overhead        : 1065.35
 
-Min latency (ns)                : 1165366
-Max latency (ns)                : 8539429
-Mean latency (ns)               : 1527250
-50.00 percentile latency (ns)   : 1442175
-90.00 percentile latency (ns)   : 2173312
-95.00 percentile latency (ns)   : 2342623
-97.00 percentile latency (ns)   : 2610918
-99.00 percentile latency (ns)   : 2636767
-99.90 percentile latency (ns)   : 2659445
+Min latency (ns)                : 851736
+Max latency (ns)                : 4430733
+Mean latency (ns)               : 938659
+50.00 percentile latency (ns)   : 919162
+90.00 percentile latency (ns)   : 1029191
+95.00 percentile latency (ns)   : 1136557
+97.00 percentile latency (ns)   : 1188814
+99.00 percentile latency (ns)   : 1208542
+99.90 percentile latency (ns)   : 1222619
 
 ================================================
 Test Parameters Used
 ================================================
 samples_per_query : 1
-target_qps : 1635.94
+target_qps : 2658.27
 target_latency (ns): 0
 max_async_queries : 1
 min_duration (ms): 600000

diff --git a/...4090x2-nvidia-gpu-TensorRT-default_config/bert-99/singlestream/TEST01/verify_accuracy.txt b/...4090x2-nvidia-gpu-TensorRT-default_config/bert-99/singlestream/TEST01/verify_accuracy.txt
@@ -4,8 +4,8 @@ Reading performance mode results...
 num_acc_log_entries = 10833
 num_acc_log_duplicate_keys = 0
 num_acc_log_data_mismatch = 0
-num_perf_log_entries = 1627
-num_perf_log_qsl_idx_match = 1627
+num_perf_log_entries = 1662
+num_perf_log_qsl_idx_match = 1662
 num_perf_log_data_mismatch = 0
 num_missing_qsl_idxs = 0
 TEST PASS

diff --git a/...0x2-nvidia-gpu-TensorRT-default_config/bert-99/singlestream/TEST01/verify_performance.txt b/...0x2-nvidia-gpu-TensorRT-default_config/bert-99/singlestream/TEST01/verify_performance.txt
@@ -1,4 +1,4 @@
 Verifying performance.
-reference score = 2176028
-test score = 2174469
+reference score = 1030837
+test score = 1029636
 TEST PASS
diff --git a/...d/MLCommons/measurements/RTX4090x2-nvidia-gpu-TensorRT-default_config/README.md b/...d/MLCommons/measurements/RTX4090x2-nvidia-gpu-TensorRT-default_config/README.md
@@ -1,4 +1,4 @@
-| Model        | Scenario     |   Accuracy |   Throughput | Latency (in ms)   | Power Efficiency (in samples/J)   | TEST01   |
-|--------------|--------------|------------|--------------|-------------------|-----------------------------------|----------|
-| 3d-unet-99.9 | singlestream |    0.86236 |        2.312 | 432.596           |                                   | passed   |
-| 3d-unet-99.9 | offline      |    0.86236 |        8.324 | -                 |                                   | passed   |
+| Model   | Scenario     |   Accuracy |   Throughput | Latency (in ms)   | Power Efficiency (in samples/J)   | TEST01   |
+|---------|--------------|------------|--------------|-------------------|-----------------------------------|----------|
+| bert-99 | singlestream |    90.2668 |      969.932 | 1.031             |                                   | passed   |
+| bert-99 | offline      |    90.1528 |     8252.75  | -                 |                                   | passed   |
diff --git a/...urements/RTX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/README.md b/...urements/RTX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/README.md
@@ -38,7 +38,7 @@ Platform: RTX4090x2-nvidia-gpu-TensorRT-default_config
 Model Precision: fp16
 
 ### Accuracy Results 
-`F1`: `90.88324`, Required accuracy for closed division `>= 90.78313`
+`F1`: `90.15279`, Required accuracy for closed division `>= 89.96526`
 
 ### Performance Results 
-`Samples per second`: `3332.18`
+`Samples per second`: `8252.75`
Original file line number	Diff line number	Diff line change
		@@ -1,2 +1,2 @@

		hash=f140a7bb4620ebc9e683e6611c1efad321151c93af78c6621b0a103f4eb8cb44
		hash=b9c1f4d44fba546a1c278a6dd98967827d1c343011f23814dedbfc7d91ffc5f9
Original file line number	Diff line number	Diff line change
		@@ -1,2 +1,2 @@

		hash=e07d27d2d42fbf5d05271577e09aff1a3ffde558fb836b04be3d64a30ad109a6
		hash=d589db7eb874fc4c9753153301ee3b630928f4048e1ac13aadca74a742750002