Tracking Farouk's changes

cld/run_sim.sh

@@ -9,34 +9,33 @@ set -x
 env
 df -h
 
-export NEV=100
-export NUM=$1 #random seed
-export SAMPLE=$2 #main card
-
-#Change these as needed
-export OUTDIR=/local/joosep/cld_edm4hep/2024_05_full/
-export SIMDIR=/home/joosep/key4hep-sim/cld/CLDConfig/CLDConfig
-export WORKDIR=/scratch/local/$USER/${SAMPLE}_${SLURM_JOB_ID}
-export FULLOUTDIR=${OUTDIR}/${SAMPLE}
-
-mkdir -p $FULLOUTDIR
-
-mkdir -p $WORKDIR
-cd $WORKDIR
-
-cp $SIMDIR/${SAMPLE}.cmd card.cmd
-cp $SIMDIR/pythia.py ./
-cp $SIMDIR/cld_steer.py ./
-cp -R $SIMDIR/PandoraSettingsCLD ./
-cp -R $SIMDIR/CLDReconstruction.py ./
-
-echo "Random:seed=${NUM}" >> card.cmd
+export NEV=$1  # number of events to generate per rootfile
+export SAMPLE=$2 # main card
+export JOBID=$3 # random seed
+export TAG=$4 # output dir tag on EOS
+
+# alias for quick access of EOS directory to copy input/output files via xrootd
+export EOSDIR=/eos/user/f/fmokhtar/
+
+mkdir CLDConfig_tmp
+dir_to_bind=$(realpath CLDConfig_tmp)
+cd $dir_to_bind
+
+# copy large input files via xrootd (recommended)
+xrdcp root://eosuser.cern.ch/$EOSDIR/key4hep-sim/cld/CLDConfig/CLDConfig/${SAMPLE}.cmd card.cmd
+xrdcp root://eosuser.cern.ch/$EOSDIR/key4hep-sim/cld/CLDConfig/CLDConfig/pythia.py pythia.py
+xrdcp root://eosuser.cern.ch/$EOSDIR/key4hep-sim/cld/CLDConfig/CLDConfig/cld_steer.py cld_steer.py
+xrdcp root://eosuser.cern.ch/$EOSDIR/key4hep-sim/cld/CLDConfig/CLDConfig/CLDReconstruction.py CLDReconstruction.py
+xrdcp -r root://eosuser.cern.ch/$EOSDIR/key4hep-sim/cld/CLDConfig/CLDConfig/PandoraSettingsCLD .
+
+# update the seed in the pythia card
+echo "Random:seed=${JOBID}" >> card.cmd
 cat card.cmd
 
 echo "
 #!/bin/bash
 set -e
-source /cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh
+source /cvmfs/sw.hsf.org/key4hep/setup.sh
 env
 k4run pythia.py -n $NEV --Dumper.Filename out.hepmc --Pythia8.PythiaInterface.pythiacard card.cmd
 ddsim -I out.hepmc -N -1 -O out_SIM.root --compactFile \$K4GEO/FCCee/CLD/compact/CLD_o2_v05/CLD_o2_v05.xml --steeringFile cld_steer.py
@@ -45,12 +44,9 @@ k4run CLDReconstruction.py --inputFiles out_SIM.root --outputBasename out_RECO -
 
 cat sim.sh
 
-# singularity exec -B /cvmfs -B /scratch -B /local docker://ghcr.io/key4hep/key4hep-images/alma9:latest bash sim.sh
-singularity exec -B /cvmfs -B /scratch -B /local /home/software/singularity/alma9.simg bash sim.sh
-
-#Copy the outputs
-cp out_RECO_edm4hep.root $FULLOUTDIR/root/reco_${SAMPLE}_${NUM}.root
-bzip2 out.hepmc
-cp out.hepmc.bz2 $FULLOUTDIR/sim/sim_${SAMPLE}_${NUM}.hepmc.bz2
+# run the event generation and PF reco
+singularity exec -B /cvmfs -B $dir_to_bind docker://ghcr.io/key4hep/key4hep-images/alma9:latest bash sim.sh
 
-rm -Rf $WORKDIR
+# copy the output files to EOS
+# xrdcp out_RECO_edm4hep.root root://eosuser.cern.ch/$EOSDIR/$TAG/reco_${SAMPLE}_${JOBID}.root
+xrdcp out_RECO_edm4hep.root root://eosuser.cern.ch//eos/project/c/cern-openlab-coeraise/data/fcc/cld/$TAG/reco_${SAMPLE}_${JOBID}.root

cld/run_sim_local.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#SBATCH -p main
+#SBATCH --mem-per-cpu=6G
+#SBATCH --cpus-per-task=1
+#SBATCH -o logs/slurm-%x-%j-%N.out
+set -e
+set -x
+
+env
+df -h
+
+export NEV=$1  # number of events to generate per rootfile
+export SAMPLE=$2 # main card
+export JOBID=$3 # random seed
+
+# set the directories
+rm -rf CLDConfig_tmp
+mkdir CLDConfig_tmp
+dir_to_bind=$(realpath CLDConfig_tmp)
+cd $dir_to_bind
+
+# copy inpit files
+cp ../CLDConfig/CLDConfig/${SAMPLE}.cmd card.cmd
+cp ../CLDConfig/CLDConfig/pythia.py pythia.py
+cp ../CLDConfig/CLDConfig/cld_steer.py cld_steer.py
+cp ../CLDConfig/CLDConfig/CLDReconstruction.py CLDReconstruction.py
+cp -r ../CLDConfig/CLDConfig/PandoraSettingsCLD .
+
+# update the seed in the pythia card
+echo "Random:seed=${JOBID}" >> card.cmd
+cat card.cmd
+
+echo "
+#!/bin/bash
+set -e
+source /cvmfs/sw.hsf.org/key4hep/setup.sh
+env
+k4run pythia.py -n $NEV --Dumper.Filename out.hepmc --Pythia8.PythiaInterface.pythiacard card.cmd
+ddsim -I out.hepmc -N -1 -O out_SIM.root --compactFile \$K4GEO/FCCee/CLD/compact/CLD_o2_v05/CLD_o2_v05.xml --steeringFile cld_steer.py
+k4run CLDReconstruction.py --inputFiles out_SIM.root --outputBasename out_RECO --num-events -1
+" > sim.sh
+
+
+# echo "
+# #!/bin/bash
+# set -e
+# source /cvmfs/sw.hsf.org/key4hep/setup.sh
+# env
+# k4run pythia.py -n $NEV --Dumper.Filename out.hepmc --Pythia8.PythiaInterface.pythiacard card.cmd
+# ddsim -I out.hepmc -N -1 -O out_SIM.root --compactFile /afs/cern.ch/user/f/fmokhtar/k4geo/FCCee/ILD_FCCee/compact/ILD_FCCee_v02/ILD_FCCee_v02.xml --steeringFile cld_steer.py
+# k4run CLDReconstruction.py --inputFiles out_SIM.root --outputBasename out_RECO --num-events -1
+# " > sim.sh
+
+# cat sim.sh
+
+# run the event generation and PF reco
+singularity exec -B /cvmfs -B $dir_to_bind docker://ghcr.io/key4hep/key4hep-images/alma9:latest bash sim.sh
+
+cp out_RECO_edm4hep.root ../reco_${SAMPLE}_${JOBID}.root
+cp out.hepmc ../sim_${SAMPLE}_${JOBID}.hepmc

cld/submit.py

@@ -0,0 +1,96 @@
+#!/usr/bin/python
+
+import argparse
+import os
+
+
+def main(args):
+
+    # create logs directory in AFS
+    if not os.path.exists(f"condor/{args.tag}/logs"):
+        os.makedirs(f"condor/{args.tag}/logs")
+
+    # create output directory on EOS
+    # eosdir = "/eos/user/f/fmokhtar/"
+    eosdir = "/eos/project/c/cern-openlab-coeraise/data/fcc/cld"
+
+    if not os.path.exists(f"{eosdir}/{args.tag}/"):
+        os.makedirs(f"{eosdir}/{args.tag}/")
+
+    Nevents = args.Nevents
+    Nevents_per_file = args.Nevents_per_file
+
+    Njobs = Nevents // Nevents_per_file
+
+    for jobid in range(Njobs):  # jobid marks the random seed
+
+        jobid += args.seed_begin
+
+        # make condor submit file with the specified jobid and sample
+        localcondor = f"condor/{args.tag}/{args.sample}_{jobid}.jdl"
+
+        condor_templ_file = open("submit.templ.jdl")
+        condor_file = open(localcondor, "w")
+        for line in condor_templ_file:
+
+            line = line.replace("NEV", str(Nevents_per_file))
+            line = line.replace("SAMPLE", args.sample)
+            line = line.replace("JOBID", str(jobid))
+            line = line.replace("TAG", args.tag)
+
+            condor_file.write(line)
+
+        condor_file.close()
+        condor_templ_file.close()
+
+        # submit
+        if args.submit:
+            print("Submit ", localcondor)
+            os.system("condor_submit %s" % localcondor)
+
+
+if __name__ == "__main__":
+    """
+    python3 submit.py --sample p8_ee_tt_ecm365 --Nevents 4 --Nevents-per-file 2 --seed-begin 0 --tag Nov15 --submit      # noqa
+    """
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--sample",
+        default="",
+        help="which sample to generate",
+        type=str,
+    )
+    parser.add_argument(
+        "--Nevents",
+        default=4,
+        help="how many events to generate",
+        type=int,
+    )
+    parser.add_argument(
+        "--Nevents-per-file",
+        default=2,
+        help="how many events to store per file",
+        type=int,
+    )
+    parser.add_argument(
+        "--seed-begin",
+        default=0,
+        help="begin count from which random seed",
+        type=int,
+    )
+    parser.add_argument(
+        "--submit",
+        action="store_true",
+        help="submit jobs when created",
+    )
+    parser.add_argument(
+        "--tag",
+        default="",
+        help="dir tag",
+        type=str,
+    )
+
+    args = parser.parse_args()
+
+    main(args)

cld/submit.templ.jdl

@@ -0,0 +1,23 @@
+# job.sub
+
+# Define the executable to run
+executable = ./run_sim.sh
+
+# Arguments for the executable (if any)
+arguments = NEV SAMPLE JOBID TAG
+
+# Specify log, output, and error files for Condor to store job info
+log    = /afs/cern.ch/user/f/fmokhtar/key4hep-sim/cld/condor/TAG/logs/job_JOBID.log
+output = /afs/cern.ch/user/f/fmokhtar/key4hep-sim/cld/condor/TAG/logs/job_JOBID.out
+error  = /afs/cern.ch/user/f/fmokhtar/key4hep-sim/cld/condor/TAG/logs/job_JOBID.err
+
+# Request resources (memory, CPUs, etc.)
+request_memory = 4 GB
+request_cpus = 1
+
+WHEN_TO_TRANSFER_OUTPUT = ON_EXIT_OR_EVICT
+
++JobFlavour = "workday"
+
+# If you want to run multiple jobs, set the job count
+queue