Results from GH action on NVIDIA_RTX4090x2

mlcommons · Feb 12, 2025 · f5f2cc5 · f5f2cc5
1 parent 53fb99b
commit f5f2cc5
Show file tree

Hide file tree

Showing 45 changed files with 1,375 additions and 1,377 deletions.
diff --git a/...RTX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/accuracy.txt b/...RTX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/accuracy.txt
@@ -1,2 +1,2 @@
 
-hash=89da28c0f1949f5c7280b508800fcdf91f1ee11b77338e19c8e9fe88f3b32976
+hash=8804e325553c9d1ac5d0a0a92ca284f54f0cc31be1e2ad02a918e38fe561da2b
diff --git a/...-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/baseline_accuracy.txt b/...-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/baseline_accuracy.txt
@@ -1,4 +1,4 @@
-{"exact_match": 25.97918637653737, "f1": 28.36486682551724}
+{"exact_match": 25.76158940397351, "f1": 28.005025763005456}
 Reading examples...
 No cached features at 'eval_features.pickle'... converting from examples...
 Creating tokenizer...

diff --git a/...vidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/compliance_accuracy.txt b/...vidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/compliance_accuracy.txt
@@ -1,4 +1,4 @@
-{"exact_match": 25.97918637653737, "f1": 28.36486682551724}
+{"exact_match": 25.75212866603595, "f1": 28.003133615417944}
 Reading examples...
 Loading cached features from 'eval_features.pickle'...
 Loading LoadGen logs...

diff --git a/...idia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/mlperf_log_accuracy.json b/...idia-gpu-TensorRT-default_config/bert-99/offline/TEST01/accuracy/mlperf_log_accuracy.json
diff --git a/...pu-TensorRT-default_config/bert-99/offline/TEST01/performance/run_1/mlperf_log_detail.txt b/...pu-TensorRT-default_config/bert-99/offline/TEST01/performance/run_1/mlperf_log_detail.txt
diff --git a/...u-TensorRT-default_config/bert-99/offline/TEST01/performance/run_1/mlperf_log_summary.txt b/...u-TensorRT-default_config/bert-99/offline/TEST01/performance/run_1/mlperf_log_summary.txt
@@ -4,7 +4,7 @@ MLPerf Results Summary
 SUT name : BERT SERVER
 Scenario : Offline
 Mode     : PerformanceOnly
-Samples per second: 3340.24
+Samples per second: 8259.87
 Result is : VALID
   Min duration satisfied : Yes
   Min queries satisfied : Yes
@@ -13,21 +13,21 @@ Result is : VALID
 ================================================
 Additional Stats
 ================================================
-Min latency (ns)                : 1159040268
-Max latency (ns)                : 666945835615
-Mean latency (ns)               : 403232474534
-50.00 percentile latency (ns)   : 429289880088
-90.00 percentile latency (ns)   : 635851997336
-95.00 percentile latency (ns)   : 654331980618
-97.00 percentile latency (ns)   : 660392436756
-99.00 percentile latency (ns)   : 665221513134
-99.90 percentile latency (ns)   : 666801899902
+Min latency (ns)                : 1524467577
+Max latency (ns)                : 666632402974
+Mean latency (ns)               : 404226159439
+50.00 percentile latency (ns)   : 430317718697
+90.00 percentile latency (ns)   : 635681784311
+95.00 percentile latency (ns)   : 653957596158
+97.00 percentile latency (ns)   : 659996914472
+99.00 percentile latency (ns)   : 664820145494
+99.90 percentile latency (ns)   : 666486440295
 
 ================================================
 Test Parameters Used
 ================================================
-samples_per_query : 2227757
-target_qps : 3375.39
+samples_per_query : 5506295
+target_qps : 8342.87
 target_latency (ns): 0
 max_async_queries : 1
 min_duration (ms): 600000

diff --git a/...e/RTX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/verify_accuracy.txt b/...e/RTX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/verify_accuracy.txt
@@ -4,9 +4,9 @@ Reading performance mode results...
 num_acc_log_entries = 10833
 num_acc_log_duplicate_keys = 0
 num_acc_log_data_mismatch = 0
-num_perf_log_entries = 4085
-num_perf_log_qsl_idx_match = 4085
-num_perf_log_data_mismatch = 51
+num_perf_log_entries = 4019
+num_perf_log_qsl_idx_match = 4019
+num_perf_log_data_mismatch = 21
 num_missing_qsl_idxs = 0
 TEST FAIL
 
diff --git a/...TX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/verify_performance.txt b/...TX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/TEST01/verify_performance.txt
@@ -1,4 +1,4 @@
 Verifying performance.
-reference score = 3341.97
-test score = 3340.24
+reference score = 8260.27
+test score = 8259.87
 TEST PASS
diff --git a/...90x2-nvidia-gpu-TensorRT-default_config/bert-99/singlestream/TEST01/accuracy/accuracy.txt b/...90x2-nvidia-gpu-TensorRT-default_config/bert-99/singlestream/TEST01/accuracy/accuracy.txt
@@ -1,2 +1,2 @@
 
-hash=eacdc285ee237f075845ba90ecaae6deb1df17cc328578b4f51263cbb542687a
+hash=dacfd1d6689f286621163bbf56ea1eb66339bef5aafa2fd3fb6031e3cc3e7aa8
diff --git a/...gpu-TensorRT-default_config/bert-99/singlestream/TEST01/accuracy/mlperf_log_accuracy.json b/...gpu-TensorRT-default_config/bert-99/singlestream/TEST01/accuracy/mlperf_log_accuracy.json
diff --git a/...nsorRT-default_config/bert-99/singlestream/TEST01/performance/run_1/mlperf_log_detail.txt b/...nsorRT-default_config/bert-99/singlestream/TEST01/performance/run_1/mlperf_log_detail.txt
diff --git a/...sorRT-default_config/bert-99/singlestream/TEST01/performance/run_1/mlperf_log_summary.txt b/...sorRT-default_config/bert-99/singlestream/TEST01/performance/run_1/mlperf_log_summary.txt
@@ -4,38 +4,38 @@ MLPerf Results Summary
 SUT name : BERT SERVER
 Scenario : SingleStream
 Mode     : PerformanceOnly
-90th percentile latency (ns) : 2167934
+90th percentile latency (ns) : 1029524
 Result is : VALID
   Min duration satisfied : Yes
   Min queries satisfied : Yes
   Early stopping satisfied: Yes
 Early Stopping Result:
- * Processed at least 64 queries (392873).
- * Would discard 38848 highest latency queries.
- * Early stopping 90th percentile estimate: 2168722
- * Early stopping 99th percentile estimate: 2637029
+ * Processed at least 64 queries (633590).
+ * Would discard 62802 highest latency queries.
+ * Early stopping 90th percentile estimate: 1029959
+ * Early stopping 99th percentile estimate: 1211028
 
 ================================================
 Additional Stats
 ================================================
-QPS w/ loadgen overhead         : 654.78
-QPS w/o loadgen overhead        : 657.39
+QPS w/ loadgen overhead         : 1055.98
+QPS w/o loadgen overhead        : 1062.77
 
-Min latency (ns)                : 1165096
-Max latency (ns)                : 3286612
-Mean latency (ns)               : 1521161
-50.00 percentile latency (ns)   : 1435171
-90.00 percentile latency (ns)   : 2167934
-95.00 percentile latency (ns)   : 2314944
-97.00 percentile latency (ns)   : 2609203
-99.00 percentile latency (ns)   : 2636743
-99.90 percentile latency (ns)   : 2660607
+Min latency (ns)                : 856366
+Max latency (ns)                : 4841428
+Mean latency (ns)               : 940941
+50.00 percentile latency (ns)   : 921338
+90.00 percentile latency (ns)   : 1029524
+95.00 percentile latency (ns)   : 1137520
+97.00 percentile latency (ns)   : 1189805
+99.00 percentile latency (ns)   : 1210576
+99.90 percentile latency (ns)   : 1223839
 
 ================================================
 Test Parameters Used
 ================================================
 samples_per_query : 1
-target_qps : 1641.39
+target_qps : 2653.72
 target_latency (ns): 0
 max_async_queries : 1
 min_duration (ms): 600000

diff --git a/...4090x2-nvidia-gpu-TensorRT-default_config/bert-99/singlestream/TEST01/verify_accuracy.txt b/...4090x2-nvidia-gpu-TensorRT-default_config/bert-99/singlestream/TEST01/verify_accuracy.txt
@@ -4,8 +4,8 @@ Reading performance mode results...
 num_acc_log_entries = 10833
 num_acc_log_duplicate_keys = 0
 num_acc_log_data_mismatch = 0
-num_perf_log_entries = 1632
-num_perf_log_qsl_idx_match = 1632
+num_perf_log_entries = 1662
+num_perf_log_qsl_idx_match = 1662
 num_perf_log_data_mismatch = 0
 num_missing_qsl_idxs = 0
 TEST PASS

diff --git a/...0x2-nvidia-gpu-TensorRT-default_config/bert-99/singlestream/TEST01/verify_performance.txt b/...0x2-nvidia-gpu-TensorRT-default_config/bert-99/singlestream/TEST01/verify_performance.txt
@@ -1,4 +1,4 @@
 Verifying performance.
-reference score = 2170812
-test score = 2168722
+reference score = 1030743
+test score = 1029959
 TEST PASS
diff --git a/...d/MLCommons/measurements/RTX4090x2-nvidia-gpu-TensorRT-default_config/README.md b/...d/MLCommons/measurements/RTX4090x2-nvidia-gpu-TensorRT-default_config/README.md
@@ -1,6 +1,4 @@
-| Model    | Scenario     |   Accuracy |   Throughput | Latency (in ms)   | Power Efficiency (in samples/J)   | TEST01   | TEST04   |
-|----------|--------------|------------|--------------|-------------------|-----------------------------------|----------|----------|
-| resnet50 | multistream  |     76.064 |     15968.1  | 0.501             |                                   | passed   | passed   |
-| resnet50 | singlestream |     76.064 |      3289.47 | 0.304             |                                   | passed   | passed   |
-| resnet50 | server       |     76.078 |     73725.3  | -                 |                                   | passed   | passed   |
-| resnet50 | offline      |     76.078 |     88009.9  | -                 |                                   | passed   | passed   |
+| Model   | Scenario     |   Accuracy |   Throughput | Latency (in ms)   | Power Efficiency (in samples/J)   | TEST01   |
+|---------|--------------|------------|--------------|-------------------|-----------------------------------|----------|
+| bert-99 | singlestream |    90.2668 |      969.932 | 1.031             |                                   | passed   |
+| bert-99 | offline      |    90.1528 |     8260.27  | -                 |                                   | passed   |
diff --git a/...urements/RTX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/README.md b/...urements/RTX4090x2-nvidia-gpu-TensorRT-default_config/bert-99/offline/README.md
@@ -17,7 +17,7 @@ pip install -U mlcflow
 
 mlc rm cache -f
 
-mlc pull repo mlcommons@mlperf-automations --checkout=0e0f74d2ce0795a81963d4afd8942ec9e431d73c
+mlc pull repo mlcommons@mlperf-automations --checkout=4be38a9703985e7e5949a74effeeda6b2b8910f3
 
 
 ```
@@ -38,7 +38,7 @@ Platform: RTX4090x2-nvidia-gpu-TensorRT-default_config
 Model Precision: fp16
 
 ### Accuracy Results 
-`F1`: `90.88324`, Required accuracy for closed division `>= 90.78313`
+`F1`: `90.15279`, Required accuracy for closed division `>= 89.96526`
 
 ### Performance Results 
-`Samples per second`: `3341.97`
+`Samples per second`: `8260.27`
Original file line number	Diff line number	Diff line change
		@@ -1,2 +1,2 @@

		hash=89da28c0f1949f5c7280b508800fcdf91f1ee11b77338e19c8e9fe88f3b32976
		hash=8804e325553c9d1ac5d0a0a92ca284f54f0cc31be1e2ad02a918e38fe561da2b
Original file line number	Diff line number	Diff line change
		@@ -1,2 +1,2 @@

		hash=eacdc285ee237f075845ba90ecaae6deb1df17cc328578b4f51263cbb542687a
		hash=dacfd1d6689f286621163bbf56ea1eb66339bef5aafa2fd3fb6031e3cc3e7aa8