diff --git a/speed-bench/m5_max.csv b/speed-bench/m5_max.csv new file mode 100644 index 000000000..c82e3d2f7 --- /dev/null +++ b/speed-bench/m5_max.csv @@ -0,0 +1,33 @@ +ctx_tokens,prefill_tokens,prefill_tps,gen_tokens,gen_tps,gen_first_ms,gen_steady_tokens,gen_steady_tps,kvcache_bytes +2048,2048,340.39,128,35.71,222.193,127,38.36,52184460 +4096,2048,330.58,128,31.41,28.997,127,31.79,80373132 +6144,2048,314.84,128,31.11,34.204,127,31.53,108561804 +8192,2048,291.24,128,30.78,35.113,127,31.18,136750476 +10240,2048,285.02,128,29.01,36.375,127,29.36,164939148 +12288,2048,270.08,128,28.72,37.404,127,29.07,193127820 +14336,2048,264.94,128,28.45,37.171,127,28.80,221316492 +16384,2048,258.35,128,27.89,37.315,127,28.21,249505164 +18432,2048,249.85,128,27.38,37.870,127,27.69,277693836 +20480,2048,244.84,128,27.32,38.804,127,27.64,305882508 +22528,2048,242.83,128,27.21,39.589,127,27.53,334071180 +24576,2048,243.17,128,27.31,38.695,127,27.63,362259852 +26624,2048,240.21,128,27.30,39.717,127,27.62,390448524 +28672,2048,241.31,128,27.23,40.005,127,27.54,418637196 +30720,2048,240.38,128,27.08,41.883,127,27.40,446825868 +32768,2048,239.28,128,27.02,41.824,127,27.34,475014540 +34816,2048,236.05,128,26.89,41.559,127,27.20,503203212 +36864,2048,234.73,128,26.91,41.064,127,27.22,531391884 +38912,2048,231.17,128,26.84,40.504,127,27.15,559580556 +40960,2048,229.98,128,26.77,40.915,127,27.08,587769228 +43008,2048,226.64,128,26.65,40.556,127,26.95,615957900 +45056,2048,223.89,128,26.60,41.169,127,26.91,644146572 +47104,2048,222.37,128,26.31,42.368,127,26.61,672335244 +49152,2048,222.48,128,26.29,40.825,127,26.58,700523916 +51200,2048,219.38,128,26.14,42.385,127,26.43,728712588 +53248,2048,217.92,128,25.79,41.816,127,26.07,756901260 +55296,2048,216.08,128,25.74,41.436,127,26.02,785089932 +57344,2048,215.59,128,25.58,40.894,127,25.85,813278604 +59392,2048,213.70,128,25.68,41.430,127,25.96,841467276 +61440,2048,211.78,128,25.62,41.186,127,25.89,869655948 +63488,2048,210.33,128,25.37,41.711,127,25.64,897844620 +65536,2048,210.42,128,25.01,43.268,127,25.28,0 diff --git a/speed-bench/m5_max_ts.svg b/speed-bench/m5_max_ts.svg new file mode 100644 index 000000000..3bb3b8102 --- /dev/null +++ b/speed-bench/m5_max_ts.svg @@ -0,0 +1,48 @@ + + + + +M5 Max t/s + +0 + +100 + +200 + +300 + +400 +0 +10 +20 +30 +40 + +0 + +20k + +40k + +60k + + + +ctx size +prefill t/s +generation t/s + + + + +prefill + +generation +