Repository navigation
Expand file tree
/
Copy pathmain.jl
More file actions
476 lines (425 loc) · 20.3 KB
/
Copy pathmain.jl
File metadata and controls
476 lines (425 loc) · 20.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
#! /usr/bin/env -S julia --color=yes --startup-file=no
# -*- coding: utf-8 -*-
#
# Gonzalez discrete-gradient time integration of the Landau collision
# operator. Refactored entry point with CLI configuration:
#
# julia --project=. main.jl parameters_default.jl
# julia --project=. main.jl parameters_picard.jl
# julia --project=. main.jl parameters_default.jl \
# --N_STEPS=200 --use_anderson=false --suffix=picard_short
#
# ARGS[1] : path to a preset file that defines `PARAMS::SimParameters`
# ARGS[2:] : zero or more `--key=value` overrides applied on top of PARAMS
#
# Diagnostics pushed to CSV every step:
# iter : Picard-map evaluations used (Anderson or Newton–Krylov)
# residual : ‖G(v) − v‖₂ at the converged iterate
# fp_minus_fs : ‖f_s − f_p‖₂ (histogram-based projection-error norm)
# neg_part : ∫ max(−f_s, 0) dv (Gibbs negative-part L¹)
include("MantisWrappers.jl")
using .MantisWrappers
using Random
using LinearAlgebra: ldiv!
# The rest of the driver, included into `Main` so the `julia main.jl <preset>`
# entry point and the runtime GPU swap below keep working unchanged.
include("io.jl") # rclone mirror, params record, checkpoints, cons CSV
include("solver.jl") # Picard map + Anderson solve, CPU/GPU hot-loop hooks
include("plots.jl") # fs_snapshot CSV + diagnostics PNGs
function run_simulation(p::SimParameters; resume = nothing)
print_summary(p)
Random.seed!(p.seed)
# Select entropy/seed integrand variant for compute_entropy / compute_r!.
USE_LOGSQ[] = p.use_logsq
ws = build_workspace(p)
println("Workspace: n_dofs=$(ws.n_dofs) n_elements=$(ws.n_elements)")
# ---- State init: either fresh sample or resume from checkpoint ----
cons_csv = "conservation_history_$(p.suffix).csv"
# Per-step solver cost: `inner` is the restart count (:anderson), Newton steps
# (:newton) or cheap inner maps (:defect); times in seconds, measured in process
# (the log is block-buffered and cannot be used for timing).
stats_csv = "solver_stats_$(p.suffix).csv"
snap_csv = "particle_snapshots_$(p.suffix).csv"
start_step = 0
t_start = 0.0 # physical time at start_step
local v_particles, w_particles, f_coeffs
local entropy_history, energy_history, momentum_history
local iter_history, res_history, fp_l2_history, neg_history
if resume !== nothing
ckpt = load_checkpoint(p.suffix, resume)
start_step = ckpt.step
v_particles = ckpt.v_particles
w_particles = ckpt.w_particles
f_coeffs = ckpt.f_coeffs
entropy_history = ckpt.entropy_history
energy_history = ckpt.energy_history
momentum_history = ckpt.momentum_history
iter_history = ckpt.iter_history
res_history = ckpt.res_history
fp_l2_history = ckpt.fp_l2_history
neg_history = ckpt.neg_history
copy!(Random.default_rng(), ckpt.rng_state)
size(v_particles, 1) == p.N_PARTICLES || error(
"Checkpoint N_PARTICLES=$(size(v_particles,1)) ≠ preset N_PARTICLES=$(p.N_PARTICLES)")
length(f_coeffs) == ws.n_dofs || error(
"Checkpoint n_dofs=$(length(f_coeffs)) ≠ workspace n_dofs=$(ws.n_dofs); mesh changed?")
start_step < p.N_STEPS || error(
"Checkpoint step=$start_step ≥ N_STEPS=$(p.N_STEPS); nothing to do")
# The CSV may predate the `dt` column; normalise it before reading
# t_start out of it or appending rows in the new layout.
migrate_cons_csv!(cons_csv)
# Physical time across the resume: the checkpoint if it has one, else
# the migrated CSV, else assume DT never changed up to here.
t_start = hasproperty(ckpt, :t) ? ckpt.t :
something(cons_time_at(cons_csv, start_step),
start_step * p.DT)
println("Resuming from step $start_step, t=$t_start " *
"(running through $(p.N_STEPS) at DT=$(p.DT))")
else
v_particles = zeros(p.N_PARTICLES, 2)
if p.v1_peak != 0.0
# Bimodal in v₁ (ported from the bimodal-v1 branch): balanced 50/50
# split centered at ±v1_peak so the net v₁ momentum is exactly zero
# regardless of N. v₂ stays N(0,σ2²).
half = p.N_PARTICLES ÷ 2
centers = [fill(-p.v1_peak, half); fill(p.v1_peak, p.N_PARTICLES - half)]
v_particles[:, 1] .= centers .+ p.σ1 .* randn(p.N_PARTICLES)
else
v_particles[:, 1] .= p.σ1 * randn(p.N_PARTICLES)
end
v_particles[:, 2] .= p.σ2 * randn(p.N_PARTICLES)
w_particles = fill(1.0 / p.N_PARTICLES, p.N_PARTICLES)
f_coeffs = zeros(ws.n_dofs)
l2_project!(ws, f_coeffs, v_particles, w_particles)
entropy_history = Float64[]
energy_history = Float64[]
momentum_history = NTuple{2, Float64}[]
iter_history = Int[]
res_history = Float64[]
fp_l2_history = Float64[]
neg_history = Float64[]
f_s0 = build_field(ws, f_coeffs)
push!(entropy_history, compute_entropy(ws, f_s0))
push!(energy_history, compute_energy(v_particles, w_particles))
push!(momentum_history, compute_momentum(v_particles, w_particles))
println("Initial S_h = $(entropy_history[end])")
println("Initial E = $(energy_history[end])")
println("Initial P = $(momentum_history[end])")
println("Initial ‖f_s − f_p‖₂ = " *
string(compute_fs_minus_fp_l2(ws, f_s0, v_particles, w_particles)))
println("Initial ∫max(−f_s,0) = " *
string(compute_negative_part_l1(ws, f_s0)))
end
f_s = build_field(ws, f_coeffs)
r_vec = zeros(ws.n_dofs)
L_vec = zeros(ws.n_dofs)
G = zeros(p.N_PARTICLES, 2)
dot_v = zeros(p.N_PARTICLES, 2)
v1 = copy(v_particles)
v_mid = similar(v_particles)
dv = similar(v_particles)
dS_mid = zeros(p.N_PARTICLES, 2)
G_eff = zeros(p.N_PARTICLES, 2)
f_buf = zeros(ws.n_dofs)
Gv = zeros(p.N_PARTICLES, 2)
r_curr = zeros(p.N_PARTICLES, 2)
r_prev = zeros(p.N_PARTICLES, 2)
Gv_prev = zeros(p.N_PARTICLES, 2)
v_old_buf = zeros(p.N_PARTICLES, 2)
ΔF = zeros(2 * p.N_PARTICLES, p.m_anderson)
ΔG = zeros(2 * p.N_PARTICLES, p.m_anderson)
p.solver in (:anderson, :newton, :defect) || error("unknown solver=$(p.solver)")
if p.solver === :defect
p.collision_model == :landau || error("solver=:defect is Landau-only")
p.use_gpu && p.gpu_fp32 &&
error("solver=:defect needs the FP64 pair kernel (gpu_fp32=false)")
end
dc = p.solver === :defect ?
(A = zeros(p.N_PARTICLES, 3), Gk = zeros(p.N_PARTICLES, 2),
Φu = zeros(p.N_PARTICLES, 2)) : nothing
nk = p.solver === :newton ? NKWorkspace(2 * p.N_PARTICLES, p.nk_krylov_max) :
nothing
nk_fd_h = p.nk_fd_h > 0 ? p.nk_fd_h : (p.use_gpu && p.gpu_fp32 ? 1e-5 : 1e-6)
# NN warm start (stateless — resume-safe without checkpoint changes).
p.warmstart in (:euler, :nn) || error("unknown warmstart=$(p.warmstart)")
nn_model = p.warmstart === :nn ? load_warmstart_model(p.nn_weights) : nothing
# Per-step training dump for the NN warm-start pipeline (crash-safe append).
dump_io = p.dump_training ?
open_training_dump("training_dump_$(p.suffix).bin",
p.N_PARTICLES, ws.n_dofs, p.DT;
append = (start_step > 0)) : nothing
# Snapshot every 25 steps (plus final step if not already a multiple of 25).
# Crash-safe: conservation + particles appended per-step / per-snapshot so a
# killed run still leaves usable data through the last completed step.
snapshot_steps = Set(0:p.snap_every:p.N_STEPS)
push!(snapshot_steps, p.N_STEPS)
snapshots_v = Dict{Int, Matrix{Float64}}()
if start_step == 0
snapshots_v[0] = copy(v_particles)
save_fs_snapshot(ws, p.suffix, 0, f_coeffs)
plot_fs_diagnostics(ws, f_coeffs, p.suffix, 0)
stats_io = open(stats_csv, "w")
println(stats_io, join(STATS_COLS, ','))
flush(stats_io)
cons_io = open(cons_csv, "w")
println(cons_io, join(CONS_COLS, ','))
println(cons_io,
"0,0.0,$(entropy_history[1]),$(energy_history[1])," *
"$(momentum_history[1][1]),$(momentum_history[1][2])," *
"0,0.0,0.0,0.0,0.0,0.0")
flush(cons_io)
rclone_upload(p.suffix, cons_csv)
snap_io = open(snap_csv, "w")
println(snap_io, "step,time,particle_idx,v1,v2")
for i in axes(v_particles, 1)
println(snap_io, "0,0.0,$i,$(v_particles[i, 1]),$(v_particles[i, 2])")
end
flush(snap_io)
# Write a step-0 checkpoint so future runs can resume even before any
# time-stepping completed (cheap and uniform).
save_checkpoint(p.suffix, 0, 0.0, v_particles, w_particles, f_coeffs,
entropy_history, energy_history, momentum_history,
iter_history, res_history, fp_l2_history, neg_history,
copy(Random.default_rng()))
else
# Resume: keep existing CSV rows ≤ start_step, append from now on.
truncate_csv_after(cons_csv, start_step)
truncate_csv_after(snap_csv, start_step)
cons_io = open(cons_csv, "a")
snap_io = open(snap_csv, "a")
if isfile(stats_csv)
truncate_csv_after(stats_csv, start_step)
stats_io = open(stats_csv, "a")
else
stats_io = open(stats_csv, "w")
println(stats_io, join(STATS_COLS, ','))
end
end
for step in (start_step + 1):p.N_STEPS
t_step0 = time_ns()
S0 = entropy_history[end]
# Explicit predictor for the Anderson initial guess. Landau uses the
# FEM-projected log-gradient G = ∇(M⁻¹r); LB uses the direct clamped
# ∇f_s/f_s (consistent with its picard map — FEM seed destabilises LB).
if p.collision_model == :lb
Base.invokelatest(LOGGRAD_FN[], ws, G, v_particles, f_coeffs)
n_lb, U1_lb, U2_lb, Q_lb = compute_moments(v_particles, w_particles)
A1_lb, A2_lb, B_lb = compute_drift_multipliers(v_particles, w_particles,
G, n_lb, U1_lb, U2_lb, Q_lb)
compute_LB_velocity!(dot_v, v_particles, G, A1_lb, A2_lb, B_lb, p.nu)
else
compute_r!(ws, r_vec, f_s)
ldiv!(L_vec, ws.M_lu, r_vec)
Base.invokelatest(COMPG_FN[], ws, G, v_particles, L_vec)
Base.invokelatest(COLLISION_FN[], ws, dot_v, v_particles, w_particles, G)
end
@. v1 = v_particles + p.DT * dot_v
# Dump the pure pre-solve state (before any NN correction): the δ
# label is reconstructed offline from the next record's v.
dump_io !== nothing &&
write_dump_record(dump_io, step, v_particles, dot_v, G, L_vec)
nn_model !== nothing &&
nn_warmstart_correct!(v1, nn_model, v_particles, dot_v, G,
w_particles, ws.bp1, ws.bp2, p.DT)
iter, res_final, n_rs, r0_init = if p.solver === :anderson
step_anderson!(ws,
v1, v_particles, w_particles, S0, p.DT,
v_mid, dv, dS_mid, G_eff, dot_v, f_buf,
r_vec, L_vec, G,
Gv, r_curr, r_prev, Gv_prev, v_old_buf, ΔF, ΔG;
m = p.m_anderson, max_iter = p.max_iter, tol = p.tol,
abs_floor = p.abs_floor,
stag_window = p.stag_window,
stag_rel_tol = p.stag_rel_tol,
damp_decay_start = p.damp_decay_start,
damp_decay_factor = p.damp_decay_factor,
damping = p.damping, use_anderson = p.use_anderson,
use_gonzalez = p.use_gonzalez,
exit_picard_step = p.exit_picard_step,
verbose = (step <= start_step + 3))
elseif p.solver === :newton
step_newton!(ws,
v1, v_particles, w_particles, S0, p.DT,
v_mid, dv, dS_mid, G_eff, dot_v, f_buf,
r_vec, L_vec, G, Gv, nk;
max_iter = p.max_iter, tol = p.tol, abs_floor = p.abs_floor,
fd_h = nk_fd_h, fd_rel = p.nk_fd_rel, eta_max = p.nk_eta_max,
use_gonzalez = p.use_gonzalez,
exit_picard_step = p.exit_picard_step,
verbose = (step <= start_step + 3))
else
step_defect!(ws,
v1, v_particles, w_particles, S0, p.DT,
v_mid, dv, dS_mid, G_eff, dot_v, f_buf,
r_vec, L_vec, G,
Gv, r_curr, r_prev, Gv_prev, v_old_buf, ΔF, ΔG, dc.A, dc.Gk, dc.Φu;
m = p.m_anderson, max_iter = p.max_iter, tol = p.tol,
abs_floor = p.abs_floor, damping = p.damping,
eta = p.dc_eta, max_inner = p.dc_max_inner,
stag_window = p.dc_stag_window, stag_rel_tol = p.stag_rel_tol,
use_gonzalez = p.use_gonzalez,
verbose = (step <= start_step + 3))
end
t_solve = (time_ns() - t_step0) / 1e9
v_particles .= v1
l2_project!(ws, f_coeffs, v_particles, w_particles)
f_s = build_field(ws, f_coeffs)
push!(entropy_history, compute_entropy(ws, f_s))
push!(energy_history, compute_energy(v_particles, w_particles))
push!(momentum_history, compute_momentum(v_particles, w_particles))
push!(iter_history, iter)
push!(res_history, res_final)
push!(fp_l2_history, compute_fs_minus_fp_l2(ws, f_s, v_particles, w_particles))
push!(neg_history, compute_negative_part_l1(ws, f_s))
# Accumulated from t_start, so a resume at a different DT extends the
# history instead of rescaling it.
t = t_start + (step - start_step) * p.DT
# Append this step's conservation row (crash-safe).
let P = momentum_history[end]
println(cons_io,
"$step,$t,$(entropy_history[end]),$(energy_history[end])," *
"$(P[1]),$(P[2]),$iter,$res_final," *
"$(fp_l2_history[end]),$(neg_history[end]),$r0_init,$(p.DT)")
flush(cons_io)
end
if step in snapshot_steps
snapshots_v[step] = copy(v_particles)
save_fs_snapshot(ws, p.suffix, step, f_coeffs)
plot_fs_diagnostics(ws, f_coeffs, p.suffix, step)
# `time` here is derived from `step`, and these files run to GB, so
# a resume does not rewrite them: rows a legacy run already wrote
# keep their `step * DT`. Take physical time from the conservation
# CSV, which the resume does migrate.
for i in axes(v_particles, 1)
println(snap_io,
"$step,$t,$i,$(v_particles[i, 1]),$(v_particles[i, 2])")
end
flush(snap_io)
save_checkpoint(p.suffix, step, t, v_particles, w_particles, f_coeffs,
entropy_history, energy_history, momentum_history,
iter_history, res_history, fp_l2_history, neg_history,
copy(Random.default_rng()))
# Mirror the growing conservation CSV at snapshot cadence (not every
# step — that would spawn an rclone process per timestep).
rclone_upload(p.suffix, cons_csv)
rclone_upload(p.suffix, stats_csv)
end
println(stats_io, "$step,$iter,$n_rs,$t_solve,$((time_ns() - t_step0) / 1e9)")
flush(stats_io)
step % 25 == 0 &&
println("Step $step/$(p.N_STEPS) iter=$iter " *
(p.solver === :anderson ? "rs=" : p.solver === :newton ? "newton=" : "inner=") *
"$n_rs" *
" ‖r‖=$(round(res_final; sigdigits=3))" *
" ‖f_s−f_p‖=$(round(fp_l2_history[end]; sigdigits=4))" *
" neg=$(round(neg_history[end]; sigdigits=4))" *
" S=$(round(entropy_history[end]; digits=6))" *
" E=$(round(energy_history[end]; digits=8))")
end
# CSVs already streamed per-step / per-snapshot above. Just close.
dump_io !== nothing && close(dump_io)
close(cons_io)
close(snap_io)
close(stats_io)
# Final mirror so the last steps (if not a multiple of 25) reach S3 too.
rclone_upload(p.suffix, cons_csv)
rclone_upload(p.suffix, stats_csv)
println("Saved $cons_csv")
println("Saved $snap_csv")
plot_run_dashboard(ws,
entropy_history, energy_history, momentum_history,
iter_history, res_history, fp_l2_history,
neg_history, p.suffix)
return (; entropy_history, energy_history, momentum_history,
iter_history, res_history, fp_l2_history, neg_history,
snapshots_v,
label = (p.solver === :newton ? "Newton–Krylov" :
p.solver === :defect ? "defect correction" :
p.use_anderson ? "Anderson(m=$(p.m_anderson))" : "Picard"))
end
"""
enable_gpu!(p)
Load the CUDA kernels and repoint the hot-loop hooks ([`COLLISION_FN`](@ref) and
friends) at them. Called at run time, so every call through the hooks must go via
`invokelatest`. Shared by `main` and scripts/check_metric_gpu.jl.
"""
function enable_gpu!(p::SimParameters)
println("GPU enabled — loading CUDA…")
# collision_gpu.jl provides _upload_col! (shared staging helper) plus the
# O(N²) Landau kernels; projection_gpu.jl provides the P_DEG=2 particle↔
# spline kernels used by BOTH operators.
include(joinpath(@__DIR__, "collision_gpu.jl"))
if p.P_DEG == 2
include(joinpath(@__DIR__, "projection_gpu.jl"))
L2PROJ_FN[] = getglobal(Main, :l2_project_gpu!)
COMPG_FN[] = getglobal(Main, :compute_G_gpu!)
println("GPU projection chain enabled (P_DEG=2 kernels)")
else
@warn "use_gpu: projection kernels are P_DEG=2-specialized; " *
"projection stays on CPU for P_DEG=$(p.P_DEG)"
end
if p.collision_model == :lb
# LB has no O(N²) sum: GPU accelerates the projection + log-gradient
# gather (the ~40 ms/iter bulk); the O(N) drift stays on CPU.
if p.P_DEG == 2
LOGGRAD_FN[] = getglobal(Main, :eval_loggrad_gpu!)
println("GPU LB log-gradient enabled")
end
else
if p.gpu_fp32
println(" ⚠ FP32 collision kernel (conservation experiment)")
COLLISION_FN[] = getglobal(Main, :compute_collision_gpu32!)
else
COLLISION_FN[] = getglobal(Main, :compute_collision_gpu!)
COLLMETRIC_FN[] = getglobal(Main, :compute_collision_metric_gpu!)
end
end
return nothing
end
function main(args = ARGS)
if isempty(args)
preset = "parameters_default.jl"
overrides = String[]
else
preset = args[1]
overrides = collect(String, args[2:end])
end
isfile(preset) || error("Preset file not found: $preset")
# Strip --resume=<step|auto> out of overrides before parse_overrides sees it
# (it's not a SimParameters field). Accepts an integer step number or the
# literal `auto` to pick the highest-step checkpoint for this suffix.
resume = nothing
overrides = filter(overrides) do tok
if startswith(tok, "--resume=")
val = tok[(length("--resume=") + 1):end]
resume = (val == "auto") ? :auto : parse(Int, val)
return false
end
return true
end
println("Loading preset: $preset")
# The preset file ends by binding PARAMS = SimParameters(...). `include`
# returns the last expression's value, so we capture PARAMS without
# relying on module globals.
params_loaded = include(joinpath(@__DIR__, preset))
p = parse_overrides(params_loaded::SimParameters, overrides)
p.use_gpu && enable_gpu!(p)
params_file = "params_$(p.suffix).jl"
save_params(params_file, p)
rclone_upload(p.suffix, params_file)
res = run_simulation(p; resume = resume)
if isempty(res.iter_history)
println("\n--- No new steps run (already at N_STEPS) ---")
else
a = sum(res.iter_history) / length(res.iter_history)
println("\n--- Inner-iter summary ---")
println("$(res.label): avg=$(round(a; digits=2)) max=$(maximum(res.iter_history))" *
" steps=$(length(res.iter_history))")
end
end
# Run only when executed as a script (`julia main.jl ...`), so the test suite and
# the REPL can `include` this file to reach the functions above without starting
# a simulation.
if abspath(PROGRAM_FILE) == @__FILE__
main()
end