note about dropout

rasbt · rasbt · commit 85827e0a0b22 · 2024-06-19T17:37:48.000-05:00
diff --git a/ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py b/ch05/03_bonus_pretraining_on_gutenberg/pretraining_simple.py
@@ -180,7 +180,7 @@ def train_model_simple(model, optimizer, device, n_epochs,
             "emb_dim": 12,           # Embedding dimension
             "n_heads": 2,            # Number of attention heads
             "n_layers": 2,           # Number of layers
-            "drop_rate": 0.0,        # Dropout rate
+            "drop_rate": 0.0,        # Dropout rate, deactivated via 0.0 as dropout in LLMs is not recommended anymore
             "qkv_bias": False        # Query-key-value bias
         }
 

Original file line number	Diff line number	Diff line change
`@@ -180,7 +180,7 @@ def train_model_simple(model, optimizer, device, n_epochs,`
`180`	`180`	`"emb_dim": 12, # Embedding dimension`
`181`	`181`	`"n_heads": 2, # Number of attention heads`
`182`	`182`	`"n_layers": 2, # Number of layers`
`183`		`- "drop_rate": 0.0, # Dropout rate`
	`183`	`+ "drop_rate": 0.0, # Dropout rate, deactivated via 0.0 as dropout in LLMs is not recommended anymore`
`184`	`184`	`"qkv_bias": False # Query-key-value bias`
`185`	`185`	`}`
`186`	`186`