Skip to content

Commit 99aead8

Browse files
changliu777changliu777
committed
add files for traverse compiling and regtest
1 parent 87d1602 commit 99aead8

7 files changed

Lines changed: 226 additions & 0 deletions

File tree

‎unstructured/regtest/KPRAD_2D/base/batchjob.traverse‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
#SBATCH -A pppl
3+
#SBATCH -n 48
4+
#SBATCH -J M3DC1_regtest_KPRAD_2D
5+
#SBATCH -t 00:30:00
6+
#SBATCH -o C1stdout
7+
8+
touch started
9+
10+
#create_fixed_mesh.sh AnalyticModel 0.06
11+
#echo $? > mesh_created
12+
13+
PARTS=48
14+
$M3DC1_MPIRUN -n $PARTS split_smb analytic-2K.smb part.smb $PARTS
15+
echo $? > mesh_partitioned
16+
17+
$M3DC1_MPIRUN -n 48 m3dc1_2d #-pc_factor_mat_solver_type mumps -mat_mumps_icntl_14 100
18+
19+
touch finished

‎unstructured/regtest/KPRAD_restart/base/batchjob.traverse‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash
2+
#SBATCH -A pppl
3+
#SBATCH -n 48
4+
#SBATCH -J M3DC1_regtest_KPRAD_2D
5+
#SBATCH -t 00:30:00
6+
#SBATCH -o C1stdout
7+
8+
touch started
9+
10+
#create_fixed_mesh.sh AnalyticModel 0.06
11+
#echo $? > mesh_created
12+
13+
PARTS=48
14+
$M3DC1_MPIRUN -n $PARTS split_smb analytic-2K.smb part.smb $PARTS
15+
echo $? > mesh_partitioned
16+
17+
cp C1input.1 C1input
18+
$M3DC1_MPIRUN -n 48 m3dc1_2d #-pc_factor_mat_solver_type mumps -mat_mumps_icntl_14 100
19+
20+
cp C1input.2 C1input
21+
$M3DC1_MPIRUN -n 48 m3dc1_2d #-pc_factor_mat_solver_type mumps -mat_mumps_icntl_14 100
22+
23+
touch finished

‎unstructured/regtest/RMP/base/batchjob.traverse‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
#SBATCH -A pppl
3+
#SBATCH -n 16
4+
#SBATCH -J M3DC1_regtest_RMP
5+
#SBATCH -t 0:10:00
6+
#SBATCH -o C1stdout
7+
8+
touch started
9+
10+
#create_mesh.sh
11+
#echo $? > mesh_created
12+
13+
PARTS=$SLURM_NTASKS
14+
$M3DC1_MPIRUN -n $PARTS split_smb diiid-0.02-2.5-4.0-4K.smb part.smb $PARTS
15+
echo $? > mesh_partitioned
16+
17+
$M3DC1_MPIRUN -n $SLURM_NTASKS m3dc1_2d_complex -pc_factor_mat_solver_type mumps
18+
19+
touch finished

‎unstructured/regtest/RMP_nonlin/base/batchjob.traverse‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
#SBATCH -A pppl
3+
#SBATCH -n 64
4+
#SBATCH -J M3DC1_regtest_RMP_nonlin
5+
#SBATCH -t 0:30:00
6+
#SBATCH -o C1stdout
7+
8+
touch started
9+
10+
# partition mesh
11+
PARTS=16
12+
$M3DC1_MPIRUN -n $PARTS split_smb circle-0.10-0.0-0.0-1K.smb part.smb $PARTS
13+
echo $? > mesh_partitioned
14+
15+
# Run M3D-C1
16+
$M3DC1_MPIRUN -n 64 m3dc1_3d -ipetsc -options_file options_bjacobi.type_superludist
17+
18+
touch finished

‎unstructured/regtest/adapt/base/batchjob.traverse‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/bash
2+
#SBATCH -A pppl
3+
#SBATCH -n 16
4+
#SBATCH -J m3dc1_regtest_adapt
5+
#SBATCH -t 0:10:00
6+
#SBATCH -o C1stdout
7+
8+
touch started
9+
10+
# partition mesh
11+
part=$(which part_mesh.sh)
12+
source $part circle-0.02-0.0-0.0-2K.smb $SLURM_NTASKS
13+
echo $? > mesh_partitioned
14+
15+
# Run M3D-C1
16+
$M3DC1_MPIRUN -n $SLURM_NTASKS m3dc1_2d
17+
$M3DC1_MPIRUN -n $SLURM_NTASKS collapse circle-0.02-0.0-0.0.dmg ts0-adapted.smb mesh.smb $SLURM_NTASKS
18+
19+
if ! [ -f mesh0.smb ]; then
20+
echo 0 > C1ke
21+
fi
22+
23+
touch finished

‎unstructured/regtest/pellet/base/batchjob.traverse‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
#SBATCH -A pppl
3+
#SBATCH -n 64
4+
#SBATCH -J M3DC1_regtest_pellet
5+
#SBATCH -t 0:30:00
6+
#SBATCH -o C1stdout
7+
8+
touch started
9+
10+
# partition mesh
11+
PARTS=16
12+
$M3DC1_MPIRUN -n $PARTS split_smb analytic-2K.smb part.smb $PARTS
13+
echo $? > mesh_partitioned
14+
15+
# Run M3D-C1
16+
$M3DC1_MPIRUN -n 64 m3dc1_3d -ipetsc -options_file options_bjacobi.type_superludist
17+
18+
touch finished

‎unstructured/traverse_gpu.mk‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
CPP = mpic++
2+
CC = mpicc
3+
F90 = mpifort
4+
F77 = mpifort
5+
LOADER = mpifort
6+
7+
OPTS := $(OPTS) -DPETSC_VERSION=990 -DUSEBLAS
8+
9+
PETSC_DIR=/projects/M3DC1/petsc
10+
11+
ifeq ($(COM), 1)
12+
PETSC_ARCH=traverse-nvidia-complex
13+
else
14+
PETSC_ARCH=traverse-nvidia-real
15+
endif
16+
17+
ifeq ($(COM), 1)
18+
M3DC1_SCOREC_LIB=-lm3dc1_scorec_complex
19+
else
20+
M3DC1_SCOREC_LIB=-lm3dc1_scorec
21+
endif
22+
23+
SCOREC_BASE_DIR=/projects/M3DC1/scorec/traverse-nvidia
24+
SCOREC_UTIL_DIR=$(SCOREC_BASE_DIR)/bin
25+
ifdef SCORECVER
26+
SCOREC_DIR=$(SCOREC_BASE_DIR)/$(SCORECVER)
27+
else
28+
SCOREC_DIR=$(SCOREC_BASE_DIR)
29+
endif
30+
31+
ZOLTAN_LIB=-L/home/liuchang/zoltan/pgi/lib -lzoltan
32+
33+
SCOREC_LIBS= -L$(SCOREC_DIR)/lib $(M3DC1_SCOREC_LIB) \
34+
-Wl,--start-group,-rpath,$(SCOREC_BASE_DIR)/lib -L$(SCOREC_BASE_DIR)/lib \
35+
-lpumi -lapf -lapf_zoltan -lgmi -llion -lma -lmds -lmth -lparma \
36+
-lpcu -lph -lsam -lspr -lcrv -Wl,--end-group
37+
38+
HDF5_DIR=/projects/M3DC1/hdf5/traverse-nvidia
39+
GSL_DIR=/projects/M3DC1/gsl/traverse
40+
FFTW_DIR=/projects/M3DC1/fftw/traverse-nvidia
41+
42+
ifeq ($(PAR), 1)
43+
OPTS := $(OPTS) -DUSEPARTICLES
44+
endif
45+
46+
PETSC_WITH_EXTERNAL_LIB = -L$(PETSC_DIR)/$(PETSC_ARCH)/lib -lpetsc -lsuperlu_dist -lsuperlu -lcmumps -ldmumps -lsmumps -lzmumps -lmumps_common -lpord -lparmetis -lmetis -lscalapack -llapack -lblas -lstdc++
47+
48+
INCLUDE := $(INCLUDE) -I$(SCOREC_DIR)/include \
49+
-I$(PETSC_DIR)/$(PETSC_ARCH)/include -I$(PETSC_DIR)/include \
50+
-I$(HDF5_DIR)/include -I$(HDF5_DIR)/include/static \
51+
-I$(FFTW_DIR)/include \
52+
-I$(GSL_DIR)/include
53+
54+
LIBS := $(LIBS) \
55+
$(SCOREC_LIBS) \
56+
$(ZOLTAN_LIB) \
57+
-L$(HDF5_DIR)/lib -lhdf5_hl_fortran -lhdf5_hl_f90cstub -lhdf5_hl -lhdf5_fortran -lhdf5_f90cstub -lhdf5 -lz \
58+
-L$(FFTW_DIR)/lib -lfftw3_mpi -lfftw3 \
59+
-L$(GSL_DIR)/lib -lgsl -lgslcblas \
60+
$(PETSC_WITH_EXTERNAL_LIB)
61+
62+
FOPTS = -c -r8 -Mpreprocess $(OPTS)
63+
64+
CCOPTS = -c $(OPTS)
65+
66+
# Optimization flags
67+
ifeq ($(OPT), 1)
68+
LDOPTS := $(LDOPTS) -fast
69+
FOPTS := $(FOPTS) -fast
70+
CCOPTS := $(CCOPTS) -fast
71+
else
72+
FOPTS := $(FOPTS) -Mbounds -Minfo=all -Mchkfpstk -Mchkstk -Mdalign -Mdclchk -Mdepchk -Miomutex -Mrecursive -Msave -Ktrap=fp -O0 -g -byteswapio
73+
CCOPTS := $(CCOPTS) -g
74+
LDOPTS := $(LDOPTS) -g
75+
endif
76+
77+
ifeq ($(OMP), 1)
78+
LDOPTS := $(LDOPTS) -mp
79+
FOPTS := $(FOPTS) -mp
80+
CCOPTS := $(CCOPTS) -mp
81+
endif
82+
83+
ifeq ($(ACC), 1)
84+
LDOPTS := $(LDOPTS) -acc -gpu=cuda11.3 -Minfo=accel
85+
FOPTS := $(FOPTS) -acc -gpu=cuda11.3 -Minfo=accel
86+
CCOPTS := $(CCOPTS) -acc -gpu=cuda11.3 -Minfo=accel
87+
endif
88+
89+
F90OPTS = $(F90FLAGS) $(FOPTS)
90+
F77OPTS = $(F77FLAGS) $(FOPTS)
91+
92+
%.o : %.cpp
93+
$(CPP) $(CCOPTS) $(INCLUDE) $< -o $@
94+
95+
%.o : %.c
96+
$(CC) $(CCOPTS) $(INCLUDE) $< -o $@
97+
98+
%.o: %.f
99+
$(F77) $(F77OPTS) $(INCLUDE) $< -o $@
100+
101+
%.o: %.F
102+
$(F77) $(F77OPTS) $(INCLUDE) $< -o $@
103+
104+
%.o: %.f90
105+
$(F90) $(F90OPTS) $(INCLUDE) $< -o $@
106+

0 commit comments

Comments
 (0)