Deployed 9c7d3b0 with MkDocs version: 1.6.1

optimization/assets/gen_images.py

@@ -279,8 +279,8 @@ def fig4_warpn_sweep(t):
     bars = ax.bar([str(n) for n in valid_ns], valid_tf,
                   color=valid_colors, width=0.7, edgecolor='none')
 
-    ax.axhline(y=337, color=t["orange"], linestyle='--', lw=1.5, alpha=0.7)
-    ax.text(0.02, 340, "v3 baseline (337)", fontsize=8,
+    ax.axhline(y=288, color=t["orange"], linestyle='--', lw=1.5, alpha=0.7)
+    ax.text(0.02, 291, "v3 baseline (288)", fontsize=8,
             color=t["orange"], transform=ax.get_yaxis_transform())
 
     ax.axhline(y=447.5, color=t["yellow"], linestyle='--', lw=1.5, alpha=0.7)
@@ -324,7 +324,7 @@ def fig5_optimization_ladder(t):
 
     labels = ["v0\nnaive", "v1\nSMEM", "v2\nTMA+WGMMA",
               "v3\nwarpspec", "v4\ntuned", "cuBLAS"]
-    values = [0.38, 1.51, 284.4, 337.0, 471.3, 447.5]
+    values = [0.38, 1.51, 284.4, 288.3, 489.9, 447.5]
     colors_l = [t["red"], t["orange"], t["blue"],
                 t["purple"], t["green"], t["yellow"]]
 
@@ -343,7 +343,7 @@ def fig5_optimization_ladder(t):
         ax.text(x_pos, bar.get_y() + bar.get_height()/2, f"{val:.1f}",
                 va='center', fontsize=10, color=t["text"], fontweight='bold')
 
-    speedups = ["", "3.9×", "188×", "1.19×", "1.40×", ""]
+    speedups = ["", "3.9×", "188×", "1.01×", "1.70×", ""]
     for i, s in enumerate(speedups):
         if s:
             ax.text(max(values[i], values[i-1]) + 40, i, s,

optimization/assets/img/optimization_ladder.png

@@ -3202,15 +3202,15 @@ <h2 id="performance-at-a-glance">Performance at a glance<a class="headerlink" hr
 </tr>
 <tr>
 <td>v3: Warp specialization</td>
-<td style="text-align: right;">3.26</td>
-<td style="text-align: right;">337.0</td>
-<td style="text-align: right;">75.3%</td>
+<td style="text-align: right;">3.81</td>
+<td style="text-align: right;">288.3</td>
+<td style="text-align: right;">64.4%</td>
 </tr>
 <tr>
 <td><strong>v4: Production-tuned</strong></td>
-<td style="text-align: right;"><strong>2.35</strong></td>
-<td style="text-align: right;"><strong>471.3</strong></td>
-<td style="text-align: right;"><strong>105.3%</strong></td>
+<td style="text-align: right;"><strong>2.24</strong></td>
+<td style="text-align: right;"><strong>489.9</strong></td>
+<td style="text-align: right;"><strong>109.5%</strong></td>
 </tr>
 <tr>
 <td>cuBLAS (reference)</td>
@@ -3220,7 +3220,7 @@ <h2 id="performance-at-a-glance">Performance at a glance<a class="headerlink" hr
 </tr>
 </tbody>
 </table>
-<p>1240× improvement from v0 to v4 in five steps. Let's see how each one works.</p>
+<p>1289× improvement from v0 to v4 in five steps. Let's see how each one works.</p>
 <hr />
 <h2 id="kernel-0-naive">Kernel 0: Naive<a class="headerlink" href="#kernel-0-naive" title="Permanent link">&para;</a></h2>
 <p><strong>File:</strong> <code>matmul_f16_v0_naive.co</code></p>
@@ -3543,7 +3543,7 @@ <h2 id="kernel-3-warp-specialization">Kernel 3: Warp Specialization<a class="hea
 <p><strong>File:</strong> <code>matmul_f16_v3_warpspec.co</code></p>
 <p>The insight from v2's profile: producer (TMA) and consumer (WGMMA) stall each other. Fix this by giving them separate warpgroups that run concurrently, connected by a software ring buffer in shared memory.</p>
 <pre class="highlight"><code class="language-choreo">// WARP_M=64, WARP_N=128, WARP_K=16
-// TILE_M=128, TILE_K=64, STAGES=1, CONSUMERS=1
+// TILE_M=64, TILE_K=64, STAGES=1, CONSUMERS=1
 
 __co__ void matmul(
     global f16 [M, K] lhs,
@@ -3553,23 +3553,26 @@ <h2 id="kernel-3-warp-specialization">Kernel 3: Warp Specialization<a class="hea
     shared event full[STAGES], empty[STAGES];
     shared f16 [TILE_M, TILE_K] lhs_s[STAGES];
     shared f16 [WARP_N, TILE_K] rhs_s[STAGES];
+    shared f16 [WARP_M, WARP_N] output_s[CONSUMERS];
 
-    // 3 warpgroups × 128 threads = 384 threads/block
-    parallel wg by 3 : group-4, t by 128 : thread {
+    // 2 warpgroups × 128 threads = 256 threads/block
+    parallel wg by 2 : group-4, t by 128 : thread {
       // ── Producer (wg=0): one thread drives TMA ──
       inthreads.async (wg == 0 &amp;&amp; t == 0) {
         foreach {iv_k} in [cdiv(K, TILE_K)] {
           stage = iv_k % STAGES;
           wait empty[stage];
           tma.copy.async&lt;full[stage]&gt;.swiz&lt;SWIZ&gt;
-            lhs.subspan(TILE_M, TILE_K).at(block_m, iv_k) =&gt; lhs_s[stage];
+            lhs.subspan(TILE_M, TILE_K)
+              .at(block_m, iv_k) =&gt; lhs_s[stage];
           tma.copy.async&lt;full[stage]&gt;.swiz&lt;SWIZ&gt;
-            rhs.subspan(WARP_N, TILE_K).at(block_n, iv_k) =&gt; rhs_s[stage];
+            rhs.subspan(WARP_N, TILE_K)
+              .at(block_n, iv_k) =&gt; rhs_s[stage];
           trigger full[stage];
         }
       }
 
-      // ── Consumers (wg&gt;=1): compute WGMMA ──
+      // ── Consumer (wg=1): compute WGMMA ──
       inthreads.async (wg &gt;= 1) {
         cidx = wg - 1;
         foreach {s} in [STAGES]
@@ -3581,17 +3584,20 @@ <h2 id="kernel-3-warp-specialization">Kernel 3: Warp Specialization<a class="hea
           wait full[stage];
 
           foreach {iv_wk} in [cdiv(TILE_K, WARP_K)] {
-            ma = mma.load.swiz&lt;SWIZ&gt; 
-                  lhs_s[stage].subspan(WARP_M, WARP_K).at(cidx, iv_wk);
-            mb = mma.load.swiz&lt;SWIZ&gt; rhs_s[stage].chunkat(_, iv_wk);
+            ma = mma.load.swiz&lt;SWIZ&gt;
+              lhs_s[stage].subspan(WARP_M, WARP_K)
+                .at(cidx, iv_wk);
+            mb = mma.load.swiz&lt;SWIZ&gt;
+              rhs_s[stage].chunkat(_, iv_wk);
             mma.row.row mc, ma, mb;
           }
+          mma.commit;
           trigger empty[stage];
         }
-        shared f16 [WARP_M, WARP_N] output_s[CONSUMERS];
         mma.store mc, output_s[cidx];
         tma.copy output_s[cidx] =&gt;
-          output.subspan(WARP_M, WARP_N).at(block_m * CONSUMERS + cidx, block_n);
+          output.subspan(WARP_M, WARP_N)
+            .at(block_m * CONSUMERS + cidx, block_n);
       }
 
     }
@@ -3639,14 +3645,14 @@ <h3 id="generated-cuda-v3-warpgroup-dispatch">Generated CUDA (v3) — warpgroup
         full[iv_k % STAGES], 1, bytes);        // trigger full
   }
 }
-if (wg &gt;= 1) {                                 // consumers
+if (wg &gt;= 1) {                                 // consumer
   for (int iv_k = 0; ...) {
     full[iv_k % STAGES].wait(...);             // wait full
     /* WGMMA (same as v2) */
     empty[iv_k % STAGES].arrive();             // trigger empty
   }
 }</code></pre>
-<p><strong>Result: ~337 TFLOPS (1.19× over v2, 75.3% of cuBLAS)</strong></p>
+<p><strong>Result: ~288 TFLOPS (1.01× over v2, 64.4% of cuBLAS)</strong></p>
 <hr />
 <h2 id="kernel-4-production-tuned">Kernel 4: Production-Tuned<a class="headerlink" href="#kernel-4-production-tuned" title="Permanent link">&para;</a></h2>
 <p><strong>File:</strong> <code>matmul_f16_v4_auto_tuned.co</code></p>
@@ -3708,7 +3714,7 @@ <h3 id="ncu-snapshot-v4">NCU snapshot (v4)<a class="headerlink" href="#ncu-snaps
 gpu__dram_throughput (% of HBM BW)  :  38.91%   ← TMA hiding all loads
 warp occupancy   (% of peak)        :  27.74%   ← ~2 blocks/SM (smem-limited)</code></pre>
 <p>SM and tensor-core utilisation are at 89.7% — the kernel is firmly in the compute-bound regime. DRAM at 39% means TMA is successfully prefetching ahead. Warp occupancy at 27.7% reflects the shared memory footprint (~80 KB per block) allowing 2 concurrent blocks per SM on H800.</p>
-<p><strong>Result: ~471 TFLOPS (1.40× over v3, 105% of cuBLAS on this GPU)</strong></p>
+<p><strong>Result: ~490 TFLOPS (1.70× over v3, 109% of cuBLAS on this GPU)</strong></p>
 <hr />
 <h2 id="the-full-ncu-picture">The full NCU picture<a class="headerlink" href="#the-full-ncu-picture" title="Permanent link">&para;</a></h2>
 <p>Profiled with a single kernel launch (<code>ncu --launch-count 1</code>):</p>
@@ -3754,15 +3760,15 @@ <h2 id="the-full-ncu-picture">The full NCU picture<a class="headerlink" href="#t
 <td style="text-align: right;">56.13</td>
 <td style="text-align: right;">9,418,787</td>
 <td style="text-align: right;">—</td>
-<td style="text-align: right;">337.0</td>
+<td style="text-align: right;">288.3</td>
 </tr>
 <tr>
 <td>v4 tuned</td>
 <td style="text-align: right;">38.91</td>
 <td style="text-align: right;">89.68</td>
 <td style="text-align: right;">9,492,372</td>
 <td style="text-align: right;">—</td>
-<td style="text-align: right;">471.3</td>
+<td style="text-align: right;">489.9</td>
 </tr>
 </tbody>
 </table>
@@ -3784,7 +3790,7 @@ <h2 id="the-full-ncu-picture">The full NCU picture<a class="headerlink" href="#t
     &lt;compiled-kernel-binary&gt;</code></pre>
 <hr />
 <h2 id="the-tuning-journey-v3-v4">The Tuning Journey: v3 → v4<a class="headerlink" href="#the-tuning-journey-v3-v4" title="Permanent link">&para;</a></h2>
-<p>v4 was not designed upfront — it was found by a systematic 28-iteration parameter sweep starting from v3. Here is the actual log. This is the core of the tutorial: the techniques above give you a structurally correct kernel, but the last 40% comes from tuning.</p>
+<p>v4 was not designed upfront — it was found by a systematic 28-iteration parameter sweep starting from v3. Here is the actual log. This is the core of the tutorial: the techniques above give you a structurally correct kernel, but the last 70% comes from tuning.</p>
 <h3 id="baseline-v3-stages1-consumers1-warp_n128">Baseline: v3 (STAGES=1, CONSUMERS=1, WARP_N=128)<a class="headerlink" href="#baseline-v3-stages1-consumers1-warp_n128" title="Permanent link">&para;</a></h3>
 <p>NCU on the v3 baseline reveals the kernel is <strong>latency-bound</strong>:</p>
 <pre class="highlight"><code>sm__throughput:          56%   ← SM underutilized
@@ -3807,7 +3813,7 @@ <h3 id="phase-1-pipeline-architecture-iter000-iter003">Phase 1 — Pipeline Arch
 <tr>
 <td>iter000</td>
 <td>baseline (STAGES=1, CONS=1)</td>
-<td style="text-align: right;">337</td>
+<td style="text-align: right;">288</td>
 <td>latency-bound</td>
 </tr>
 <tr>
@@ -3926,9 +3932,9 @@ <h2 id="sota-comparison">SOTA comparison<a class="headerlink" href="#sota-compar
 </tr>
 <tr>
 <td><strong>Croqtile v4 (tuned)</strong></td>
-<td style="text-align: right;"><strong>2.35</strong></td>
-<td style="text-align: right;"><strong>471.3</strong></td>
-<td style="text-align: right;"><strong>105.3%</strong></td>
+<td style="text-align: right;"><strong>2.24</strong></td>
+<td style="text-align: right;"><strong>489.9</strong></td>
+<td style="text-align: right;"><strong>109.5%</strong></td>
 </tr>
 </tbody>
 </table>
@@ -3941,8 +3947,8 @@ <h2 id="sota-comparison">SOTA comparison<a class="headerlink" href="#sota-compar
 <p>All three can be expressed in Croqtile — the production-grade kernels in the repository implement them.</p>
 <hr />
 <h2 id="the-optimization-ladder">The optimization ladder<a class="headerlink" href="#the-optimization-ladder" title="Permanent link">&para;</a></h2>
-<p><img alt="Optimization ladder: TFLOPS progression from v0 (0.38) through v4 (471), compared to cuBLAS (447)" src="../assets/img/optimization_ladder.png#only-dark" />
-<img alt="Optimization ladder: TFLOPS progression from v0 (0.38) through v4 (471), compared to cuBLAS (447)" src="../assets/img/optimization_ladder_light.png#only-light" /></p>
+<p><img alt="Optimization ladder: TFLOPS progression from v0 (0.38) through v4 (490), compared to cuBLAS (447)" src="../assets/img/optimization_ladder.png#only-dark" />
+<img alt="Optimization ladder: TFLOPS progression from v0 (0.38) through v4 (490), compared to cuBLAS (447)" src="../assets/img/optimization_ladder_light.png#only-light" /></p>
 <table>
 <thead>
 <tr>
@@ -3969,13 +3975,13 @@ <h2 id="the-optimization-ladder">The optimization ladder<a class="headerlink" hr
 <td>v2 → v3</td>
 <td>Warp specialization</td>
 <td><code>inthreads.async</code>, <code>shared event</code>, <code>wait/trigger</code></td>
-<td style="text-align: right;">1.19×</td>
+<td style="text-align: right;">1.01×</td>
 </tr>
 <tr>
 <td>v3 → v4</td>
 <td>Pipeline tuning</td>
 <td>STAGES=2, CONSUMERS=2, WARP_N=192 (28-iter sweep)</td>
-<td style="text-align: right;">1.40×</td>
+<td style="text-align: right;">1.70×</td>
 </tr>
 </tbody>
 </table>