Repository navigation
Expand file tree
/
Copy pathdebug_ttt.sh
More file actions
450 lines (405 loc) · 30.3 KB
/
Copy pathdebug_ttt.sh
File metadata and controls
450 lines (405 loc) · 30.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
export repo_dir="${repo_dir:-${PWD}}"
export home_dir="${home_dir:-${repo_dir}}"
bash "${repo_dir}/flame_script/env/setup_env.sh"
cd "${repo_dir}/flame_script/flame_task"
export HF_HOME=${home_dir}/.cache/huggingface
export TORCH_HOME=${home_dir}/.cache/torch
export HUGGINGFACE_HUB_CACHE=${home_dir}/.cache/huggingface/hub
export TRANSFORMERS_CACHE=${home_dir}/.cache/huggingface/hub
# Required training data paths:
# export DATA_10B_PATH=/path/to/10b/training/data
# export DATA_100B_PATH=/path/to/100b/training/data
# Parameter-count note:
# - Config/script names such as 90m / 310m / 1.2B are historical
# backbone-scale labels. They roughly refer to the non-vocabulary model
# body: transformer blocks, token mixer, channel mixer, and norms.
# - Paper scales such as 160M / 410M / 1.45B report total model parameters.
# They include the vocabulary token embedding and the untied LM head
# (tie_word_embeddings=false), which add about 2 * vocab_size * hidden_size
# parameters on top of the backbone.
# Therefore: 90m -> 160M, 310m -> 410M, and 1.2B -> 1.45B in the paper.
################################################################################
# paper scale-up baseline: LLaMA
################################################################################
# 310m / 100B / 4k context
# config_dir=xmixers/llama/naive_half_rope
# bash script.sh 1 false $config_dir llama_half_rope_310m 8 2 4096 false llama_half_rope 100000 16 3e-4 10000 100b
# 1.2B / 100B / 4k context
# config_dir=xmixers/llama/naive_half_rope
# bash script.sh 1 false $config_dir llama_half_rope_1_2b 4 2 4096 false llama_half_rope 100000 16 3e-4 10000 100b
# related 20k / 2k ablation
# 90m
# config_dir=xmixers/llama/naive_half_rope
# bash script.sh 1 false $config_dir llama_half_rope_90m 32 1 2048 false llama_half_rope 20000 192
# 310m
# config_dir=xmixers/llama/naive_half_rope
# bash script.sh 1 false $config_dir llama_half_rope_310m 16 2 2048 false llama_half_rope 20000 128
# 1.2B
# config_dir=xmixers/llama/naive_half_rope
# bash script.sh 1 false $config_dir llama_half_rope_1_2b 16 2 2048 false llama_half_rope 20000 64
################################################################################
# paper scale-up baseline: GDN (Gated DeltaNet)
################################################################################
# 310m / 100B / 4k context
# config_dir=xmixers/deltanet/scalar_decay_deltanet_offset
# bash script.sh 1 false $config_dir scalar_decay_deltanet_offset_310m 8 2 4096 false deltanet 100000 32 3e-4 10000 100b
# 1.2B / 100B / 4k context
# config_dir=xmixers/deltanet/scalar_decay_deltanet_offset
# bash script.sh 1 false $config_dir scalar_decay_deltanet_offset_1_2b 4 2 4096 false deltanet 100000 32 3e-4 10000 100b
# related 20k / 2k ablation
# 90m
# config_dir=xmixers/deltanet/scalar_decay_deltanet_offset
# bash script.sh 1 false $config_dir scalar_decay_deltanet_offset_90m 32 1 2048 false deltanet 20000 96
# 310m
# config_dir=xmixers/deltanet/scalar_decay_deltanet_offset
# bash script.sh 1 false $config_dir scalar_decay_deltanet_offset_310m 16 2 2048 false deltanet 20000 64
# 1.2B
# config_dir=xmixers/deltanet/scalar_decay_deltanet_offset
# bash script.sh 1 false $config_dir scalar_decay_deltanet_offset_1_2b 16 2 2048 false deltanet 20000 32
################################################################################
# paper scale-up: MSE loss, M, scalar decay, no mean scaling
################################################################################
# 310m / 100B / 4k context
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_sinlr_310m_official_init 8 2 4096 false ttt_linear_mse_no_norm_sd 100000 64 3e-4 10000 100b
# 1.2B / 100B / 4k context
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_sinlr_1_2b_official_init 4 2 4096 false ttt_linear_mse_no_norm_sd 100000 32 3e-4 10000 100b
# related 20k / 2k ablation
# graph memory + sinlr, official init, mse loss without mean + scalar decay
# 90m
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_sinlr_90m_official_init 32 1 2048 false ttt_linear_mse_no_norm_sd 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_sinlr_310m_official_init 16 2 2048 false ttt_linear_mse_no_norm_sd 20000 128
# 1.2B
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_sinlr_1_2b_official_init 16 2 2048 false ttt_linear_mse_no_norm_sd 20000 64
################################################################################
# paper scale-up: Inner-product loss, M, scalar decay, no mean scaling
################################################################################
# 310m / 100B / 4k context
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sd_sinlr_310m_official_init 8 2 4096 false ttt_linear_inner_no_norm_sd 100000 64 3e-4 10000 100b
# 1.2B / 100B / 4k context
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sd_sinlr_1_2b_official_init 4 2 4096 false ttt_linear_inner_no_norm_sd 100000 32 3e-4 10000 100b
# related 20k / 2k ablation
# graph memory + sinlr, official init, inner loss without mean + scalar decay
# 90m
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sd_sinlr_90m_official_init 32 1 2048 false ttt_linear_inner_no_norm_sd 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sd_sinlr_310m_official_init 16 2 2048 false ttt_linear_inner_no_norm_sd 20000 128
# 1.2B
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sd_sinlr_1_2b_official_init 16 2 2048 false ttt_linear_inner_no_norm_sd 20000 64
################################################################################
# paper scale-up: MSE loss, M-SiLU, scalar decay, no mean scaling
################################################################################
# 310m / 100B / 4k context
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_silu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_silu_single_sinlr_310m_official_init 8 2 4096 false ttt_linear_mse_no_norm_sd_silu_single 100000 64 3e-4 10000 100b
# 1.2B / 100B / 4k context
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_silu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_silu_single_sinlr_1_2b_official_init 4 2 4096 false ttt_linear_mse_no_norm_sd_silu_single 100000 32 3e-4 10000 100b
# related 20k / 2k ablation
# graph memory + sinlr, official init, mse loss without mean + scalar decay + silu
# 90m
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_silu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_silu_single_sinlr_90m_official_init 32 1 2048 false ttt_linear_mse_no_norm_sd_silu_single 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_silu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_silu_single_sinlr_310m_official_init 16 2 2048 false ttt_linear_mse_no_norm_sd_silu_single 20000 128
# 1.2B
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_silu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_silu_single_sinlr_1_2b_official_init 16 2 2048 false ttt_linear_mse_no_norm_sd_silu_single 20000 64
################################################################################
# paper scale-up: Inner-product loss, M-SiLU, scalar decay, no mean scaling
################################################################################
# 310m / 100B / 4k context
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sd_silu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sd_silu_single_sinlr_310m_official_init 8 2 4096 false ttt_linear_inner_no_norm_sd_silu_single 100000 64 3e-4 10000 100b
# 1.2B / 100B / 4k context
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sd_silu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sd_silu_single_sinlr_1_2b_official_init 4 2 4096 false ttt_linear_inner_no_norm_sd_silu_single 100000 32 3e-4 10000 100b
# related 20k / 2k ablation
# graph memory + sinlr, official init, inner loss without mean + scalar decay + silu
# 90m
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sd_silu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sd_silu_single_sinlr_90m_official_init 32 1 2048 false ttt_linear_inner_no_norm_sd_silu_single 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sd_silu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sd_silu_single_sinlr_310m_official_init 16 2 2048 false ttt_linear_inner_no_norm_sd_silu_single 20000 128
# 1.2B
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sd_silu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sd_silu_single_sinlr_1_2b_official_init 16 2 2048 false ttt_linear_inner_no_norm_sd_silu_single 20000 64
################################################################################
# graph memory + sinlr, official init
################################################################################
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_mean_sd_silu_norm_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_mean_sd_silu_norm_sinlr_official_init_90m 32 1 2048 false ttt_mlp_mse_mean_sd_silu_norm 20000 192
################################################################################
# graph memory + sinlr, official init, no mean
################################################################################
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_silu_norm_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_silu_norm_sinlr_official_init_90m 32 1 2048 false ttt_mlp_mse_sd_silu_norm 20000 192
################################################################################
# graph memory + sinlr1e4, official init, ttt_rmsnorm_eps=0.5
################################################################################
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_mean_sd_silu_norm_sinlr1e4_official_init_eps05
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_mean_sd_silu_norm_sinlr1e4_official_init_90m_eps05 32 1 2048 false ttt_mlp_mse_mean_sd_silu_norm 20000 192
################################################################################
# graph memory + sinlr1e4, official init, no mean, ttt_rmsnorm_eps=0.5
################################################################################
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_silu_norm_sinlr1e4_official_init_eps05
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_silu_norm_sinlr1e4_official_init_90m_eps05 32 1 2048 false ttt_mlp_mse_sd_silu_norm 20000 192
################################################################################
# graph memory pre-norm residual version, official init, mean + sinlr1e4, ttt_rmsnorm_eps=0.5
################################################################################
# 结构：linear -> silu -> linear -> add(input, branch) -> norm，scalar decay
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_mean_sd_silu_residual_pre_norm_sinlr1e4_official_init_eps05
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_mean_sd_silu_residual_pre_norm_sinlr1e4_official_init_90m_eps05 32 1 2048 false ttt_mlp_graph_mse_mean_sd_silu_residual_pre_norm 20000 192
################################################################################
# graph memory pre-norm residual version, official init, no mean + sinlr1e4, ttt_rmsnorm_eps=0.5
################################################################################
# 结构：linear -> silu -> linear -> add(input, branch) -> norm，scalar decay
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_silu_residual_pre_norm_sinlr1e4_official_init_eps05
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_silu_residual_pre_norm_sinlr1e4_official_init_90m_eps05 32 1 2048 false ttt_mlp_graph_mse_sd_silu_residual_pre_norm 20000 192
################################################################################
# graph memory pre-norm residual version, official init, no mean + sinlr1e4
################################################################################
# 结构：linear -> silu -> linear -> add(input, branch) -> norm，scalar decay
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_silu_residual_pre_norm_sinlr1e4_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_silu_residual_pre_norm_sinlr1e4_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_sd_silu_residual_pre_norm 20000 192
################################################################################
# graph memory residual act version, official init, no mean + sinlr1e4
################################################################################
# 结构：linear -> silu -> linear -> add(input, branch) -> silu，scalar decay
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_silu_residual_act_sinlr1e4_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_silu_residual_act_sinlr1e4_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_sd_silu_residual_act 20000 192
################################################################################
# graph memory residual act version, official init, mean + sinlr1e4
################################################################################
# 结构：linear -> silu -> linear -> add(input, branch) -> silu，scalar decay
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_mean_sd_silu_residual_act_sinlr1e4_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_mean_sd_silu_residual_act_sinlr1e4_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_mean_sd_silu_residual_act 20000 192
################################################################################
# graph memory + sinlr, official init, scalar decay
################################################################################
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_mean_sd_linear_linear_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_mean_sd_linear_linear_sinlr_official_init_90m 32 1 2048 false ttt_mlp_mse_mean_sd_linear_linear 20000 192
# 310m
# config_dir=modular_ttt/ttt_mlp_graph_mse_mean_sd_linear_linear_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_mean_sd_linear_linear_sinlr_official_init_310m 16 2 2048 false ttt_mlp_mse_mean_sd_linear_linear 20000 128
################################################################################
# graph memory + sinlr, official init, scalar decay, no mean
################################################################################
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_linear_linear_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_linear_linear_sinlr_official_init_90m 32 1 2048 false ttt_mlp_mse_sd_linear_linear 20000 192
# 310m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_linear_linear_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_linear_linear_sinlr_official_init_310m 16 2 2048 false ttt_mlp_mse_sd_linear_linear 20000 128
################################################################################
# graph memory + sinlr, scalar decay, official init
################################################################################
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_mean_sd_linear_linear_norm_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_mean_sd_linear_linear_norm_sinlr_official_init_90m 32 1 2048 false ttt_mlp_mse_mean_sd_linear_linear_norm 20000 192
################################################################################
# graph memory + sinlr, scalar decay, official init, no mean
################################################################################
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_linear_linear_norm_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_linear_linear_norm_sinlr_official_init_90m 32 1 2048 false ttt_mlp_mse_sd_linear_linear_norm 20000 192
################################################################################
# graph memory + sinlr, scalar decay, official init
################################################################################
# 结构：linear -> linear -> silu
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_mean_sd_linear_linear_silu_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_mean_sd_linear_linear_silu_sinlr_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_mean_sd_linear_linear_silu 20000 192
################################################################################
# graph memory + sinlr, scalar decay, official init, no mean
################################################################################
# 结构：linear -> linear -> silu
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_linear_linear_silu_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_linear_linear_silu_sinlr_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_sd_linear_linear_silu 20000 192
################################################################################
# graph memory + sinlr, scalar decay, official init
################################################################################
# 结构：linear -> norm -> linear
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_mean_sd_linear_norm_linear_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_mean_sd_linear_norm_linear_sinlr_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_mean_sd_linear_norm_linear 20000 192
################################################################################
# graph memory + sinlr, scalar decay, official init, no mean
################################################################################
# 结构：linear -> norm -> linear
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_linear_norm_linear_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_linear_norm_linear_sinlr_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_sd_linear_norm_linear 20000 192
################################################################################
# graph memory + sinlr, scalar decay, official init
################################################################################
# 结构：linear -> silu -> linear
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_mean_sd_linear_silu_linear_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_mean_sd_linear_silu_linear_sinlr_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_mean_sd_linear_silu_linear 20000 192
################################################################################
# graph memory + sinlr, scalar decay, official init, no mean
################################################################################
# 结构：linear -> silu -> linear
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_linear_silu_linear_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_linear_silu_linear_sinlr_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_sd_linear_silu_linear 20000 192
################################################################################
# graph memory + sinlr, scalar decay, official init
################################################################################
# 结构：linear -> silu -> linear -> silu
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_mean_sd_linear_silu_linear_silu_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_mean_sd_linear_silu_linear_silu_sinlr_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_mean_sd_linear_silu_linear_silu 20000 192
################################################################################
# graph memory + sinlr, scalar decay, official init, no mean
################################################################################
# 结构：linear -> silu -> linear -> silu
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_linear_silu_linear_silu_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_linear_silu_linear_silu_sinlr_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_sd_linear_silu_linear_silu 20000 192
################################################################################
# graph memory + sinlr, scalar decay, official init
################################################################################
# 结构：linear -> norm -> linear -> norm
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_mean_sd_linear_norm_linear_norm_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_mean_sd_linear_norm_linear_norm_sinlr_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_mean_sd_linear_norm_linear_norm 20000 192
################################################################################
# graph memory + sinlr, scalar decay, official init, no mean
################################################################################
# 结构：linear -> norm -> linear -> norm
# 90m
# config_dir=modular_ttt/ttt_mlp_graph_mse_sd_linear_norm_linear_norm_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_mlp_graph_mse_sd_linear_norm_linear_norm_sinlr_official_init_90m 32 1 2048 false ttt_mlp_graph_mse_sd_linear_norm_linear_norm 20000 192
################################################################################
# graph memory + sinlr, official init, vector decay, mse loss without mean
################################################################################
# 90m
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_vd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_vd_sinlr_90m_official_init 32 1 2048 false ttt_linear_mse_no_norm_vd 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_vd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_vd_sinlr_310m_official_init 16 2 2048 false ttt_linear_mse_no_norm_vd 20000 128
################################################################################
# graph memory + sinlr, official init, vector decay, inner loss without mean
################################################################################
# 90m
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_vd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_vd_sinlr_90m_official_init 32 1 2048 false ttt_linear_inner_no_norm_vd 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_vd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_vd_sinlr_310m_official_init 16 2 2048 false ttt_linear_inner_no_norm_vd 20000 128
################################################################################
# graph memory version, official init, mse loss without mean
################################################################################
# 90m
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_90m_official_init 32 1 2048 false ttt_linear_mse_no_norm_sd 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_310m_official_init 16 2 2048 false ttt_linear_mse_no_norm_sd 20000 128
################################################################################
# graph memory + sinlr, official init, mse loss without mean
################################################################################
# 90m
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sinlr_90m_official_init 32 1 2048 false ttt_linear_mse_no_norm 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sinlr_310m_official_init 16 2 2048 false ttt_linear_mse_no_norm 20000 128
################################################################################
# graph memory + sinlr, official init, inner loss without mean
################################################################################
# 90m
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sinlr_90m_official_init 32 1 2048 false ttt_linear_inner_no_norm 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sinlr_310m_official_init 16 2 2048 false ttt_linear_inner_no_norm 20000 128
################################################################################
# graph memory + sinlr, official init, mse loss without mean + scalar decay + gelu_tanh
################################################################################
# 90m
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_gelu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_gelu_single_sinlr_90m_official_init 32 1 2048 false ttt_linear_mse_no_norm_sd_gelu_single 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_gelu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_gelu_single_sinlr_310m_official_init 16 2 2048 false ttt_linear_mse_no_norm_sd_gelu_single 20000 128
# 1.2B
# config_dir=modular_ttt/ttt_linear_graph_mse_no_norm_sd_gelu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_no_norm_sd_gelu_single_sinlr_1_2b_official_init 16 2 2048 false ttt_linear_mse_no_norm_sd_gelu_single 20000 64
################################################################################
# graph memory + sinlr, official init, inner loss without mean + scalar decay + gelu_tanh
################################################################################
# 90m
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sd_gelu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sd_gelu_single_sinlr_90m_official_init 32 1 2048 false ttt_linear_inner_no_norm_sd_gelu_single 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_inner_no_norm_sd_gelu_single_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_no_norm_sd_gelu_single_sinlr_310m_official_init 16 2 2048 false ttt_linear_inner_no_norm_sd_gelu_single 20000 128
################################################################################
# graph memory + sinlr, official init, l1 loss without mean + scalar decay
################################################################################
# 90m
# config_dir=modular_ttt/ttt_linear_graph_l1_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_l1_no_norm_sd_sinlr_90m_official_init 32 1 2048 false ttt_linear_l1_no_norm_sd 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_l1_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_l1_no_norm_sd_sinlr_310m_official_init 16 2 2048 false ttt_linear_l1_no_norm_sd 20000 128
################################################################################
# graph memory + sinlr, official init, rmse loss without mean + scalar decay
################################################################################
# 90m
# config_dir=modular_ttt/ttt_linear_graph_rmse_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_rmse_no_norm_sd_sinlr_90m_official_init 32 1 2048 false ttt_linear_rmse_no_norm_sd 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_rmse_no_norm_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_rmse_no_norm_sd_sinlr_310m_official_init 16 2 2048 false ttt_linear_rmse_no_norm_sd 20000 128
################################################################################
# graph memory + sinlr, official init, mse loss without mean + norm(no fast weight)
################################################################################
# 90m
# config_dir=modular_ttt/ttt_linear_graph_mse_norm_no_f_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_norm_no_f_sd_sinlr_90m_official_init 32 1 2048 false ttt_linear_mse_norm_no_f_sd 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_mse_norm_no_f_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_norm_no_f_sd_sinlr_310m_official_init 16 2 2048 false ttt_linear_mse_norm_no_f_sd 20000 128
# 1.2B
# config_dir=modular_ttt/ttt_linear_graph_mse_norm_no_f_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_mse_norm_no_f_sd_sinlr_1_2b_official_init 16 2 2048 false ttt_linear_mse_norm_no_f_sd 20000 64
################################################################################
# graph memory + sinlr, official init, inner loss without mean + norm(no fast weight)
################################################################################
# 90m
# config_dir=modular_ttt/ttt_linear_graph_inner_norm_no_f_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_norm_no_f_sd_sinlr_90m_official_init 32 1 2048 false ttt_linear_inner_norm_no_f_sd 20000 192
# 310m
# config_dir=modular_ttt/ttt_linear_graph_inner_norm_no_f_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_norm_no_f_sd_sinlr_310m_official_init 16 2 2048 false ttt_linear_inner_norm_no_f_sd 20000 128
# 1.2B
# config_dir=modular_ttt/ttt_linear_graph_inner_norm_no_f_sd_sinlr_official_init
# bash script.sh 1 false $config_dir ttt_linear_graph_inner_norm_no_f_sd_sinlr_1_2b_official_init 16 2 2048 false ttt_linear_inner_norm_no_f_sd 20000 64