wtf

ngc92 · ngc92 · commit 24ea11fe5675 · 2025-11-23T02:54:41.000+01:00
diff --git a/src/models/llama_model.cpp b/src/models/llama_model.cpp
@@ -424,8 +424,8 @@ void LLamaModel::backward(Tensor inputs, Tensor targets, NCCLCommunicator& comm,
                              rs->Encoded, weights.LN1_w, rs->Acts[l].LN1_Rstd, nullptr, B, T, C, rs->DeviceProp, main_stream);
         }
         Parameters->release_block(l, main_stream);
-        Grads->notify_block(l, main_stream, comm);
         CUDA_CHECK(cudaEventRecord(test_event, comm.stream()));
+        Grads->notify_block(l, main_stream, comm);
     }
 
     auto& d_emb = Grads->get_embeddings_full(main_stream, comm, accumulate);

Original file line number	Diff line number	Diff line change
`@@ -424,8 +424,8 @@ void LLamaModel::backward(Tensor inputs, Tensor targets, NCCLCommunicator& comm,`
`424`	`424`	`rs->Encoded, weights.LN1_w, rs->Acts[l].LN1_Rstd, nullptr, B, T, C, rs->DeviceProp, main_stream);`
`425`	`425`	`}`
`426`	`426`	`Parameters->release_block(l, main_stream);`
`427`		`- Grads->notify_block(l, main_stream, comm);`
`428`	`427`	`CUDA_CHECK(cudaEventRecord(test_event, comm.stream()));`
	`428`	`+ Grads->notify_block(l, main_stream, comm);`
`429`	`429`	`}`
`430`	`430`
`431`	`431`	`auto& d_emb = Grads->get_embeddings_full(main_stream, comm, accumulate);`