diff --git a/README.md b/README.md index 887b15f7d..7b573a64e 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,25 @@ Each benchmark will run until the target quality is reached and then stop, print Some these benchmarks are rather slow or take a long time to run on the reference hardware. We expect to see significant performance improvements with more hardware and optimized implementations. +# MLPerf Training v6.0 (Submission Deadline May 15, 2026) + +| Model | reference implementation | framework* | dataset | model parameter count** +| ---- | ---- | ---- | ---- | ---- +| flux.1 | [text_to_image](https://github.com/mlcommons/training/tree/master/text_to_image) | torchtitan | CC12M subset | 11.9B +| llama3.1_8b | [small_llm_pretraining](https://github.com/mlcommons/training/tree/master/small_llm_pretraining) | NeMo | C4 | 8b +| llama2_70b_lora | [llama2_70b_lora](https://github.com/mlcommons/training/tree/master/llama2_70b_lora) | pytorch | SCROLLS GovReport | 70B +| llama3.1_405b | [large_language_model_pretraining](https://github.com/mlcommons/training/tree/master/large_language_model_pretraining) | NeMo | C4 | 405B +| dlrm_dcnv2 | [recommendation_v2](https://github.com/mlcommons/training/tree/master/recommendation_v2/torchrec_dlrm) | torchrec | Criteo 3.5TB multi-hot | 167M +| gpt_oss_20b*** | TBD | TBD | TBD | TBD | +| deepseekv3*** | TBD | TBD | TBD | TBD | +| llm_reasoning*** | TBD | TBD | TBD | TBD | + +*Framework here is given for the reference implementation. Submitters are free to use their own frameworks to run the benchmark. + +**Model parameter count is not the same as active parameter that are being trained in the benchmark. + +***Work in progress so details are TBD (To be decided) + # MLPerf Training v5.1 (Submission Deadline Oct 10, 2025) | Model | reference implementation | framework* | dataset | model parameter count** diff --git a/single_stage_detector/.dockerignore b/retired_benchmarks/retinanet/.dockerignore similarity index 100% rename from single_stage_detector/.dockerignore rename to retired_benchmarks/retinanet/.dockerignore diff --git a/single_stage_detector/Dockerfile b/retired_benchmarks/retinanet/Dockerfile similarity index 100% rename from single_stage_detector/Dockerfile rename to retired_benchmarks/retinanet/Dockerfile diff --git a/single_stage_detector/README.md b/retired_benchmarks/retinanet/README.md similarity index 100% rename from single_stage_detector/README.md rename to retired_benchmarks/retinanet/README.md diff --git a/single_stage_detector/mlcube/README.md b/retired_benchmarks/retinanet/mlcube/README.md similarity index 100% rename from single_stage_detector/mlcube/README.md rename to retired_benchmarks/retinanet/mlcube/README.md diff --git a/single_stage_detector/mlcube/mlcube.yaml b/retired_benchmarks/retinanet/mlcube/mlcube.yaml similarity index 100% rename from single_stage_detector/mlcube/mlcube.yaml rename to retired_benchmarks/retinanet/mlcube/mlcube.yaml diff --git a/single_stage_detector/requirements.txt b/retired_benchmarks/retinanet/requirements.txt similarity index 100% rename from single_stage_detector/requirements.txt rename to retired_benchmarks/retinanet/requirements.txt diff --git a/single_stage_detector/scripts/backbone_to_onnx.py b/retired_benchmarks/retinanet/scripts/backbone_to_onnx.py similarity index 100% rename from single_stage_detector/scripts/backbone_to_onnx.py rename to retired_benchmarks/retinanet/scripts/backbone_to_onnx.py diff --git a/single_stage_detector/scripts/download_backbone.sh b/retired_benchmarks/retinanet/scripts/download_backbone.sh similarity index 100% rename from single_stage_detector/scripts/download_backbone.sh rename to retired_benchmarks/retinanet/scripts/download_backbone.sh diff --git a/single_stage_detector/scripts/download_coco2017.sh b/retired_benchmarks/retinanet/scripts/download_coco2017.sh similarity index 100% rename from single_stage_detector/scripts/download_coco2017.sh rename to retired_benchmarks/retinanet/scripts/download_coco2017.sh diff --git a/single_stage_detector/scripts/download_openimages_demo.sh b/retired_benchmarks/retinanet/scripts/download_openimages_demo.sh similarity index 100% rename from single_stage_detector/scripts/download_openimages_demo.sh rename to retired_benchmarks/retinanet/scripts/download_openimages_demo.sh diff --git a/single_stage_detector/scripts/download_openimages_full.sh b/retired_benchmarks/retinanet/scripts/download_openimages_full.sh similarity index 100% rename from single_stage_detector/scripts/download_openimages_full.sh rename to retired_benchmarks/retinanet/scripts/download_openimages_full.sh diff --git a/single_stage_detector/scripts/download_openimages_mlperf.sh b/retired_benchmarks/retinanet/scripts/download_openimages_mlperf.sh similarity index 100% rename from single_stage_detector/scripts/download_openimages_mlperf.sh rename to retired_benchmarks/retinanet/scripts/download_openimages_mlperf.sh diff --git a/single_stage_detector/scripts/fiftyone_openimages.py b/retired_benchmarks/retinanet/scripts/fiftyone_openimages.py similarity index 100% rename from single_stage_detector/scripts/fiftyone_openimages.py rename to retired_benchmarks/retinanet/scripts/fiftyone_openimages.py diff --git a/single_stage_detector/scripts/pth_to_onnx.py b/retired_benchmarks/retinanet/scripts/pth_to_onnx.py similarity index 100% rename from single_stage_detector/scripts/pth_to_onnx.py rename to retired_benchmarks/retinanet/scripts/pth_to_onnx.py diff --git a/single_stage_detector/scripts/pth_to_pickle.py b/retired_benchmarks/retinanet/scripts/pth_to_pickle.py similarity index 100% rename from single_stage_detector/scripts/pth_to_pickle.py rename to retired_benchmarks/retinanet/scripts/pth_to_pickle.py diff --git a/single_stage_detector/ssd/.gitignore b/retired_benchmarks/retinanet/ssd/.gitignore similarity index 100% rename from single_stage_detector/ssd/.gitignore rename to retired_benchmarks/retinanet/ssd/.gitignore diff --git a/single_stage_detector/ssd/LICENSE b/retired_benchmarks/retinanet/ssd/LICENSE similarity index 100% rename from single_stage_detector/ssd/LICENSE rename to retired_benchmarks/retinanet/ssd/LICENSE diff --git a/single_stage_detector/ssd/bind.sh b/retired_benchmarks/retinanet/ssd/bind.sh similarity index 100% rename from single_stage_detector/ssd/bind.sh rename to retired_benchmarks/retinanet/ssd/bind.sh diff --git a/single_stage_detector/ssd/check_logs.sh b/retired_benchmarks/retinanet/ssd/check_logs.sh similarity index 100% rename from single_stage_detector/ssd/check_logs.sh rename to retired_benchmarks/retinanet/ssd/check_logs.sh diff --git a/single_stage_detector/ssd/coco_eval.py b/retired_benchmarks/retinanet/ssd/coco_eval.py similarity index 100% rename from single_stage_detector/ssd/coco_eval.py rename to retired_benchmarks/retinanet/ssd/coco_eval.py diff --git a/single_stage_detector/ssd/coco_utils.py b/retired_benchmarks/retinanet/ssd/coco_utils.py similarity index 100% rename from single_stage_detector/ssd/coco_utils.py rename to retired_benchmarks/retinanet/ssd/coco_utils.py diff --git a/single_stage_detector/ssd/config_DGXA100_001x08x032.sh b/retired_benchmarks/retinanet/ssd/config_DGXA100_001x08x032.sh similarity index 100% rename from single_stage_detector/ssd/config_DGXA100_001x08x032.sh rename to retired_benchmarks/retinanet/ssd/config_DGXA100_001x08x032.sh diff --git a/single_stage_detector/ssd/config_DGXA100_002x08x016.sh b/retired_benchmarks/retinanet/ssd/config_DGXA100_002x08x016.sh similarity index 100% rename from single_stage_detector/ssd/config_DGXA100_002x08x016.sh rename to retired_benchmarks/retinanet/ssd/config_DGXA100_002x08x016.sh diff --git a/single_stage_detector/ssd/config_DGXA100_008x08x004_inference_benchmark.sh b/retired_benchmarks/retinanet/ssd/config_DGXA100_008x08x004_inference_benchmark.sh similarity index 100% rename from single_stage_detector/ssd/config_DGXA100_008x08x004_inference_benchmark.sh rename to retired_benchmarks/retinanet/ssd/config_DGXA100_008x08x004_inference_benchmark.sh diff --git a/single_stage_detector/ssd/config_DGXA100_008x08x008.sh b/retired_benchmarks/retinanet/ssd/config_DGXA100_008x08x008.sh similarity index 100% rename from single_stage_detector/ssd/config_DGXA100_008x08x008.sh rename to retired_benchmarks/retinanet/ssd/config_DGXA100_008x08x008.sh diff --git a/single_stage_detector/ssd/config_DGXA100_032x08x032.sh b/retired_benchmarks/retinanet/ssd/config_DGXA100_032x08x032.sh similarity index 100% rename from single_stage_detector/ssd/config_DGXA100_032x08x032.sh rename to retired_benchmarks/retinanet/ssd/config_DGXA100_032x08x032.sh diff --git a/single_stage_detector/ssd/engine.py b/retired_benchmarks/retinanet/ssd/engine.py similarity index 100% rename from single_stage_detector/ssd/engine.py rename to retired_benchmarks/retinanet/ssd/engine.py diff --git a/single_stage_detector/ssd/model/__init__.py b/retired_benchmarks/retinanet/ssd/model/__init__.py similarity index 100% rename from single_stage_detector/ssd/model/__init__.py rename to retired_benchmarks/retinanet/ssd/model/__init__.py diff --git a/single_stage_detector/ssd/model/anchor_utils.py b/retired_benchmarks/retinanet/ssd/model/anchor_utils.py similarity index 100% rename from single_stage_detector/ssd/model/anchor_utils.py rename to retired_benchmarks/retinanet/ssd/model/anchor_utils.py diff --git a/single_stage_detector/ssd/model/backbone_utils.py b/retired_benchmarks/retinanet/ssd/model/backbone_utils.py similarity index 100% rename from single_stage_detector/ssd/model/backbone_utils.py rename to retired_benchmarks/retinanet/ssd/model/backbone_utils.py diff --git a/single_stage_detector/ssd/model/boxes.py b/retired_benchmarks/retinanet/ssd/model/boxes.py similarity index 100% rename from single_stage_detector/ssd/model/boxes.py rename to retired_benchmarks/retinanet/ssd/model/boxes.py diff --git a/single_stage_detector/ssd/model/feature_pyramid_network.py b/retired_benchmarks/retinanet/ssd/model/feature_pyramid_network.py similarity index 100% rename from single_stage_detector/ssd/model/feature_pyramid_network.py rename to retired_benchmarks/retinanet/ssd/model/feature_pyramid_network.py diff --git a/single_stage_detector/ssd/model/focal_loss.py b/retired_benchmarks/retinanet/ssd/model/focal_loss.py similarity index 100% rename from single_stage_detector/ssd/model/focal_loss.py rename to retired_benchmarks/retinanet/ssd/model/focal_loss.py diff --git a/single_stage_detector/ssd/model/image_list.py b/retired_benchmarks/retinanet/ssd/model/image_list.py similarity index 100% rename from single_stage_detector/ssd/model/image_list.py rename to retired_benchmarks/retinanet/ssd/model/image_list.py diff --git a/single_stage_detector/ssd/model/resnet.py b/retired_benchmarks/retinanet/ssd/model/resnet.py similarity index 100% rename from single_stage_detector/ssd/model/resnet.py rename to retired_benchmarks/retinanet/ssd/model/resnet.py diff --git a/single_stage_detector/ssd/model/retinanet.py b/retired_benchmarks/retinanet/ssd/model/retinanet.py similarity index 100% rename from single_stage_detector/ssd/model/retinanet.py rename to retired_benchmarks/retinanet/ssd/model/retinanet.py diff --git a/single_stage_detector/ssd/model/roi_heads.py b/retired_benchmarks/retinanet/ssd/model/roi_heads.py similarity index 100% rename from single_stage_detector/ssd/model/roi_heads.py rename to retired_benchmarks/retinanet/ssd/model/roi_heads.py diff --git a/single_stage_detector/ssd/model/transform.py b/retired_benchmarks/retinanet/ssd/model/transform.py similarity index 100% rename from single_stage_detector/ssd/model/transform.py rename to retired_benchmarks/retinanet/ssd/model/transform.py diff --git a/single_stage_detector/ssd/model/utils.py b/retired_benchmarks/retinanet/ssd/model/utils.py similarity index 100% rename from single_stage_detector/ssd/model/utils.py rename to retired_benchmarks/retinanet/ssd/model/utils.py diff --git a/single_stage_detector/ssd/presets.py b/retired_benchmarks/retinanet/ssd/presets.py similarity index 100% rename from single_stage_detector/ssd/presets.py rename to retired_benchmarks/retinanet/ssd/presets.py diff --git a/single_stage_detector/ssd/run.sub b/retired_benchmarks/retinanet/ssd/run.sub similarity index 100% rename from single_stage_detector/ssd/run.sub rename to retired_benchmarks/retinanet/ssd/run.sub diff --git a/single_stage_detector/ssd/run_and_time.sh b/retired_benchmarks/retinanet/ssd/run_and_time.sh similarity index 100% rename from single_stage_detector/ssd/run_and_time.sh rename to retired_benchmarks/retinanet/ssd/run_and_time.sh diff --git a/single_stage_detector/ssd/run_demo.sh b/retired_benchmarks/retinanet/ssd/run_demo.sh similarity index 100% rename from single_stage_detector/ssd/run_demo.sh rename to retired_benchmarks/retinanet/ssd/run_demo.sh diff --git a/single_stage_detector/ssd/ssd_logger.py b/retired_benchmarks/retinanet/ssd/ssd_logger.py similarity index 100% rename from single_stage_detector/ssd/ssd_logger.py rename to retired_benchmarks/retinanet/ssd/ssd_logger.py diff --git a/single_stage_detector/ssd/train.py b/retired_benchmarks/retinanet/ssd/train.py similarity index 100% rename from single_stage_detector/ssd/train.py rename to retired_benchmarks/retinanet/ssd/train.py diff --git a/single_stage_detector/ssd/transforms.py b/retired_benchmarks/retinanet/ssd/transforms.py similarity index 100% rename from single_stage_detector/ssd/transforms.py rename to retired_benchmarks/retinanet/ssd/transforms.py diff --git a/single_stage_detector/ssd/utils.py b/retired_benchmarks/retinanet/ssd/utils.py similarity index 100% rename from single_stage_detector/ssd/utils.py rename to retired_benchmarks/retinanet/ssd/utils.py diff --git a/graph_neural_network/.dockerignore b/retired_benchmarks/rgat/.dockerignore similarity index 100% rename from graph_neural_network/.dockerignore rename to retired_benchmarks/rgat/.dockerignore diff --git a/graph_neural_network/Dockerfile b/retired_benchmarks/rgat/Dockerfile similarity index 100% rename from graph_neural_network/Dockerfile rename to retired_benchmarks/rgat/Dockerfile diff --git a/graph_neural_network/Dockerfile.h100 b/retired_benchmarks/rgat/Dockerfile.h100 similarity index 100% rename from graph_neural_network/Dockerfile.h100 rename to retired_benchmarks/rgat/Dockerfile.h100 diff --git a/graph_neural_network/Dockerfile_mlcube b/retired_benchmarks/rgat/Dockerfile_mlcube similarity index 100% rename from graph_neural_network/Dockerfile_mlcube rename to retired_benchmarks/rgat/Dockerfile_mlcube diff --git a/graph_neural_network/README.md b/retired_benchmarks/rgat/README.md similarity index 100% rename from graph_neural_network/README.md rename to retired_benchmarks/rgat/README.md diff --git a/graph_neural_network/build_partition_feature.py b/retired_benchmarks/rgat/build_partition_feature.py similarity index 100% rename from graph_neural_network/build_partition_feature.py rename to retired_benchmarks/rgat/build_partition_feature.py diff --git a/graph_neural_network/compress_graph.py b/retired_benchmarks/rgat/compress_graph.py similarity index 100% rename from graph_neural_network/compress_graph.py rename to retired_benchmarks/rgat/compress_graph.py diff --git a/graph_neural_network/compress_graph_demo.py b/retired_benchmarks/rgat/compress_graph_demo.py similarity index 100% rename from graph_neural_network/compress_graph_demo.py rename to retired_benchmarks/rgat/compress_graph_demo.py diff --git a/graph_neural_network/dataset.py b/retired_benchmarks/rgat/dataset.py similarity index 100% rename from graph_neural_network/dataset.py rename to retired_benchmarks/rgat/dataset.py diff --git a/graph_neural_network/dist_train_rgnn.py b/retired_benchmarks/rgat/dist_train_rgnn.py similarity index 100% rename from graph_neural_network/dist_train_rgnn.py rename to retired_benchmarks/rgat/dist_train_rgnn.py diff --git a/graph_neural_network/download.py b/retired_benchmarks/rgat/download.py similarity index 100% rename from graph_neural_network/download.py rename to retired_benchmarks/rgat/download.py diff --git a/graph_neural_network/download_data.sh b/retired_benchmarks/rgat/download_data.sh similarity index 100% rename from graph_neural_network/download_data.sh rename to retired_benchmarks/rgat/download_data.sh diff --git a/graph_neural_network/download_demo.sh b/retired_benchmarks/rgat/download_demo.sh similarity index 100% rename from graph_neural_network/download_demo.sh rename to retired_benchmarks/rgat/download_demo.sh diff --git a/graph_neural_network/download_igbh_full.sh b/retired_benchmarks/rgat/download_igbh_full.sh similarity index 100% rename from graph_neural_network/download_igbh_full.sh rename to retired_benchmarks/rgat/download_igbh_full.sh diff --git a/graph_neural_network/mlcube/.gitignore b/retired_benchmarks/rgat/mlcube/.gitignore similarity index 100% rename from graph_neural_network/mlcube/.gitignore rename to retired_benchmarks/rgat/mlcube/.gitignore diff --git a/graph_neural_network/mlcube/mlcube.yaml b/retired_benchmarks/rgat/mlcube/mlcube.yaml similarity index 100% rename from graph_neural_network/mlcube/mlcube.yaml rename to retired_benchmarks/rgat/mlcube/mlcube.yaml diff --git a/graph_neural_network/mlcube/readme.md b/retired_benchmarks/rgat/mlcube/readme.md similarity index 100% rename from graph_neural_network/mlcube/readme.md rename to retired_benchmarks/rgat/mlcube/readme.md diff --git a/graph_neural_network/mlperf_logging_utils.py b/retired_benchmarks/rgat/mlperf_logging_utils.py similarity index 100% rename from graph_neural_network/mlperf_logging_utils.py rename to retired_benchmarks/rgat/mlperf_logging_utils.py diff --git a/graph_neural_network/partition.py b/retired_benchmarks/rgat/partition.py similarity index 100% rename from graph_neural_network/partition.py rename to retired_benchmarks/rgat/partition.py diff --git a/graph_neural_network/process_data.sh b/retired_benchmarks/rgat/process_data.sh similarity index 100% rename from graph_neural_network/process_data.sh rename to retired_benchmarks/rgat/process_data.sh diff --git a/graph_neural_network/rgnn.py b/retired_benchmarks/rgat/rgnn.py similarity index 100% rename from graph_neural_network/rgnn.py rename to retired_benchmarks/rgat/rgnn.py diff --git a/graph_neural_network/run_and_time.sh b/retired_benchmarks/rgat/run_and_time.sh similarity index 100% rename from graph_neural_network/run_and_time.sh rename to retired_benchmarks/rgat/run_and_time.sh diff --git a/graph_neural_network/run_demo.sh b/retired_benchmarks/rgat/run_demo.sh similarity index 100% rename from graph_neural_network/run_demo.sh rename to retired_benchmarks/rgat/run_demo.sh diff --git a/graph_neural_network/split_seeds.py b/retired_benchmarks/rgat/split_seeds.py similarity index 100% rename from graph_neural_network/split_seeds.py rename to retired_benchmarks/rgat/split_seeds.py diff --git a/graph_neural_network/train_rgnn_multi_gpu.py b/retired_benchmarks/rgat/train_rgnn_multi_gpu.py similarity index 100% rename from graph_neural_network/train_rgnn_multi_gpu.py rename to retired_benchmarks/rgat/train_rgnn_multi_gpu.py diff --git a/graph_neural_network/utilities.py b/retired_benchmarks/rgat/utilities.py similarity index 100% rename from graph_neural_network/utilities.py rename to retired_benchmarks/rgat/utilities.py