Merge pull request #211 from soleti/poms_campaign

Created a folder with POMS configuration files

CampaignConfig/corsika.cfg

@@ -0,0 +1,307 @@
+[global]
+group = mu2e
+experiment = mu2e
+wrapper = file:///${FIFE_UTILS_DIR}/libexec/fife_wrap
+account = srsoleti
+file_type = mc
+run_type = physics
+b_name = %(project_name)s
+basename = override_me
+treename = override_me
+sam_dataset = override_me
+fcl = override_me
+numevents = override_me
+numjobs = override_me
+startevent = 1
+description = corsika_dsstop
+outdir = override_me
+logdir = override_me
+streamname = only
+fcl_list = override_me
+stage_name = override_me
+project_name = corsika_dsstop
+tardir = /pnfs/mu2e/resilient/users/%(account)/gridexport/tmp.J0XFFWlueU/
+date = 20200420
+release = Offline
+
+[env_pass]
+IFDH_DEBUG = 1
+SAM_EXPERIMENT = %(experiment)s
+
+[submit]
+debug = True
+G = %(group)s
+#N = %(numjobs)s
+e = SAM_EXPERIMENT
+e_1 = IFDH_DEBUG
+e_2 = POMS4_CAMPAIGN_NAME
+e_3 = POMS4_CAMPAIGN_STAGE_NAME
+resource-provides = usage_model=DEDICATED,OPPORTUNISTIC
+generate-email-summary = True
+expected-lifetime = 23h
+OS = SL7
+memory = 2000MB
+email-to = %(account)[email protected]
+tar_file_name = %(tardir)s/Code.tar.bz
+
+
+[job_setup]
+debug = True
+find_setups = False
+source_1 = /cvmfs/%(experiment)s.opensciencegrid.org/setup%(experiment)s-art.sh
+source_2 = ${_CONDOR_JOB_IWD}/Code/Offline/setup.sh
+setup_1 = mu2egrid
+setup_2 = dhtools
+setup_3 = ifdh_art v2_10_00 -q e19:prof
+setup_4 = mu2etools
+setup_5 = mu2efiletools
+setup_6 = corsika
+prescript_1 = LD_LIBRARY_PATH=${_CONDOR_JOB_IWD}/Code/Offline/vendor_perl/lib64/Digest/SHA:$LD_LIBRARY_PATH
+prescript_2 = PERL5LIB=${_CONDOR_JOB_IWD}/Code/Offline/vendor_perl/:$PERL5LIB
+ifdh_art = True
+
+[sam_consumer]
+limit = 1
+schema = xroot
+appvers = %(release)s
+appfamily = art
+appname = test
+
+[job_output]
+addoutput = *.art
+# rename = unique
+dest = %(outdir)s
+declare_metadata = True
+metadata_extractor = jsonMaker -x -f usr-sim
+add_location = True
+filter_metadata = checksum
+# filter_metadata = data_stream,file_format,art.first_event,art.last_event,art.process_name,art.file_format_version,art.file_format_era,checksum
+add_metadata = file_format=art
+hash = 2
+
+[job_output_2]
+addoutput = %(treename)s*.root
+# rename = unique
+dest = %(outdir)s
+hash = 2
+declare_metadata = False
+
+[job_output_1]
+addoutput = *.fcl
+;rename = unique
+dest = %(outdir)s
+declare_metadata = True
+metadata_extractor = json
+add_location = True
+filter_metadata = checksum
+# filter_metadata = data_stream,file_format,art.first_event,art.last_event,art.process_name,art.file_format_version,art.file_format_era,checksum
+add_metadata = file_format=fcl
+hash = 2
+
+[stage_gen_fcl]
+job_output_1.add_to_dataset = _poms_analysis
+job_output_1.filter_metadata = checksum,parents
+job_setup.prescript_3 = printf '#include "JobConfig/cosmic/cosmic_s1_dsstops_corsika.fcl"\n' > template.fcl
+job_setup.prescript_4 = ${_CONDOR_JOB_IWD}/Code/Offline/fileNamesGenerator.sh 2000 %(account) > filenames.txt
+job_setup.prescript_5 = generate_fcl --description=corsika_dsstops --dsconf=%(date)s --inputs=filenames.txt --merge-factor=1 --embed template.fcl
+job_setup.prescript_6 = sed -i "s/MU2EGRIDDSOWNER/%(account)s/g" 000/*.fcl
+job_setup.prescript_7 = sed -i "s/MU2EGRIDDSCONF/%(date)s/g" 000/*.fcl
+job_setup.prescript_8 = mv 000/* .
+job_setup.prescript_9 = rm template.fcl
+job_setup.ifdh_art = False
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_s1_fcl_%(date)s
+executable.name = true
+
+[stage_gen]
+# _poms_analysis is a keyword for POMS to instruct it to make its own dataset
+# and followup with it to propagate it as input of next stage
+job_output.add_to_dataset = _poms_analysis
+job_output.dataset_exclude = *truncated*
+job_output_2.add_to_dataset = %(account)_gen_root_test
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_s1_%(date)s
+job_setup.getconfig = True
+submit.n_files_per_job = 1
+sam_consumer.limit = 1
+submit.dataset = %(dataset)s
+executable.name = \\\\\${_CONDOR_JOB_IWD}/Code/Offline/getFilename.sh
+job_setup.postscript = CORSIKA_EXE=`which corsika77100Linux_QGSJET_fluka`
+job_setup.postscript_2 = DATDIR=`dirname $CORSIKA_EXE`
+job_setup.postscript_3 = sed -e "s:_DATDIR_:$DATDIR/:" -e "s:_DIRECT_:`pwd`/:" -e "s:_SEED1_:`cat seed.txt`:" -e "s:_SEED2_:`cat seed.txt`:" -e "s:_NSHOW_:2000000:" ${_CONDOR_JOB_IWD}/Code/Offline/corsika_input_nolongi_TEMPLATE > corsika_conf.txt
+job_setup.postscript_4 = cat corsika_conf.txt
+job_setup.postscript_5 = corsika77100Linux_QGSJET_fluka < corsika_conf.txt > corsika_log.txt
+job_setup.postscript_6 = mv DAT* `cat filename.txt`
+job_setup.postscript_7 = mu2e -c torun.fcl --sam-data-tier=Output:sim
+job_setup.postscript_8 = rm torun.fcl
+job_setup.multifile = False
+job_setup.setup_local = True
+
+[stage_resampler_hi_fcl]
+job_output_1.add_to_dataset = _poms_analysis
+job_setup.prescript_3 = printf '#include "JobConfig/cosmic/dsstops_resampler_hi.fcl"\n' >> template.fcl
+job_setup.prescript_4 = ${_CONDOR_JOB_IWD}/Code/Offline/listFiles.sh %(dataset)s > inputs.txt
+job_setup.prescript_5 = generate_fcl --description=corsika_resampler_hi --dsconf=%(date)s --aux=1:physics.filters.dsResample.fileNames:inputs.txt --run-number=1 --events-per-job=200000 -njobs=2000 --embed template.fcl
+job_setup.prescript_6 = sed -i "s/MU2EGRIDDSOWNER/%(account)s/g" 000/*.fcl
+job_setup.prescript_7 = sed -i "s/MU2EGRIDDSCONF/%(date)s/g" 000/*.fcl
+job_setup.prescript_8 = mv 000/* .
+job_setup.prescript_9 = rm template.fcl
+job_setup.ifdh_art = False
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_resampler_hi_fcl_%(date)s
+executable.name = true
+
+[stage_resampler_hi]
+# _poms_analysis is a keyword for POMS to instruct it to make its own dataset
+# and followup with it to propagate it as input of next stage
+job_output.add_to_dataset = _poms_analysis
+job_output.dataset_exclude = *truncated*
+job_output_2.add_to_dataset = %(account)_resampler_hi_root
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_resampler_hi_%(date)s
+job_setup.getconfig = True
+submit.n_files_per_job = 1
+sam_consumer.limit = 1
+submit.dataset = %(dataset)s
+executable.arg_1 = --sam-data-tier=Output:sim
+job_setup.multifile = False
+job_setup.setup_local = True
+
+[stage_digi_hi_fcl]
+job_output_1.add_to_dataset = _poms_analysis
+job_setup.prescript_3 = printf '#include "JobConfig/primary/CORSIKA-offspill.fcl"\n' >> template.fcl
+job_setup.prescript_4 = ${_CONDOR_JOB_IWD}/Code/Offline/listFiles.sh %(dataset)s > inputs.txt
+job_setup.prescript_5 = generate_fcl --desc=corsika_digi_hi --dsconf=%(date)s --inputs=inputs.txt --merge=10 --embed template.fcl
+job_setup.prescript_6 = sed -i "s/MU2EGRIDDSOWNER/%(account)s/g" 000/*.fcl
+job_setup.prescript_7 = sed -i "s/MU2EGRIDDSCONF/%(date)s/g" 000/*.fcl
+job_setup.prescript_8 = mv 000/* .
+job_setup.prescript_9 = rm template.fcl
+job_setup.ifdh_art = False
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_digi_hi_fcl_%(date)s
+executable.name = true
+
+[stage_digi_hi]
+# _poms_analysis is a keyword for POMS to instruct it to make its own dataset
+# and followup with it to propagate it as input of next stage
+job_output.add_to_dataset = _poms_analysis
+job_output.dataset_exclude = *truncated*
+job_output_2.add_to_dataset = %(account)_digi_hi_root
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_digi_hi_%(date)s
+job_setup.getconfig = True
+submit.n_files_per_job = 1
+sam_consumer.limit = 1
+submit.dataset = %(dataset)s
+executable.arg_1 = --sam-data-tier=Output:dig
+job_setup.multifile = False
+job_setup.setup_local = True
+
+[stage_reco_hi_fcl]
+job_output_1.add_to_dataset = _poms_analysis
+job_setup.prescript_3 = printf '#include "JobConfig/reco/CORSIKA-cosmic-general-mix.fcl"\n' >> template.fcl
+job_setup.prescript_4 = ${_CONDOR_JOB_IWD}/Code/Offline/listFiles.sh %(dataset)s > inputs.txt
+job_setup.prescript_5 = generate_fcl --desc=corsika_reco_hi --dsconf=%(date)s --inputs=inputs.txt --merge=10 --embed template.fcl
+job_setup.prescript_6 = sed -i "s/MU2EGRIDDSOWNER/%(account)s/g" 000/*.fcl
+job_setup.prescript_7 = sed -i "s/MU2EGRIDDSCONF/%(date)s/g" 000/*.fcl
+job_setup.prescript_8 = mv 000/* .
+job_setup.prescript_9 = rm template.fcl
+job_setup.ifdh_art = False
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_reco_hi_fcl_%(date)s
+executable.name = true
+
+[stage_reco_hi]
+# _poms_analysis is a keyword for POMS to instruct it to make its own dataset
+# and followup with it to propagate it as input of next stage
+job_output.add_to_dataset = _poms_analysis
+job_output.dataset_exclude = *truncated*
+job_output_2.add_to_dataset = %(account)_reco_hi_root
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_reco_hi_%(date)s
+job_setup.getconfig = True
+submit.n_files_per_job = 1
+sam_consumer.limit = 1
+submit.dataset = %(dataset)s
+executable.arg_1 = --sam-data-tier=Output:mcs
+job_setup.multifile = False
+job_setup.setup_local = True
+
+[stage_resampler_lo_fcl]
+job_output_1.add_to_dataset = _poms_analysis
+job_setup.prescript_3 = printf '#include "JobConfig/cosmic/dsstops_resampler_lo.fcl"\n' >> template.fcl
+job_setup.prescript_4 = ${_CONDOR_JOB_IWD}/Code/Offline/listFiles.sh %(dataset)s > inputs.txt
+job_setup.prescript_5 = generate_fcl --description=corsika_resampler_lo --dsconf=%(date)s --aux=1:physics.filters.dsResample.fileNames:inputs.txt --run-number=1 --events-per-job=200000 -njobs=2000 --embed template.fcl
+job_setup.prescript_6 = sed -i "s/MU2EGRIDDSOWNER/%(account)s/g" 000/*.fcl
+job_setup.prescript_7 = sed -i "s/MU2EGRIDDSCONF/%(date)s/g" 000/*.fcl
+job_setup.prescript_8 = mv 000/* .
+job_setup.prescript_9 = rm template.fcl
+job_setup.ifdh_art = False
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_resampler_lo_fcl_%(date)s
+executable.name = true
+
+[stage_resampler_lo]
+# _poms_analysis is a keyword for POMS to instruct it to make its own dataset
+# and followup with it to propagate it as input of next stage
+job_output.add_to_dataset = _poms_analysis
+job_output.dataset_exclude = *truncated*
+job_output_2.add_to_dataset = %(account)_resampler_lo_root
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_resampler_lo_%(date)s
+job_setup.getconfig = True
+submit.n_files_per_job = 1
+sam_consumer.limit = 1
+submit.dataset = %(dataset)s
+executable.arg_1 = --sam-data-tier=Output:sim
+job_setup.multifile = False
+job_setup.setup_local = True
+
+[stage_digi_lo_fcl]
+job_output_1.add_to_dataset = _poms_analysis
+job_setup.prescript_3 = printf '#include "JobConfig/primary/CORSIKA-offspill.fcl"\n' >> template.fcl
+job_setup.prescript_4 = ${_CONDOR_JOB_IWD}/Code/Offline/listFiles.sh %(dataset)s > inputs.txt
+job_setup.prescript_5 = generate_fcl --desc=corsika_digi_lo --dsconf=%(date)s --inputs=inputs.txt --merge=10 --embed template.fcl
+job_setup.prescript_6 = sed -i "s/MU2EGRIDDSOWNER/%(account)s/g" 000/*.fcl
+job_setup.prescript_7 = sed -i "s/MU2EGRIDDSCONF/%(date)s/g" 000/*.fcl
+job_setup.prescript_8 = mv 000/* .
+job_setup.prescript_9 = rm template.fcl
+job_setup.ifdh_art = False
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_digi_lo_fcl_%(date)s
+executable.name = true
+
+[stage_digi_lo]
+# _poms_analysis is a keyword for POMS to instruct it to make its own dataset
+# and followup with it to propagate it as input of next stage
+job_output.add_to_dataset = _poms_analysis
+job_output.dataset_exclude = *truncated*
+job_output_2.add_to_dataset = %(account)_digi_lo_root
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_digi_lo_%(date)s
+job_setup.getconfig = True
+submit.n_files_per_job = 1
+sam_consumer.limit = 1
+submit.dataset = %(dataset)s
+executable.arg_1 = --sam-data-tier=Output:dig
+job_setup.multifile = False
+job_setup.setup_local = True
+
+[stage_reco_lo_fcl]
+job_output_1.add_to_dataset = _poms_analysis
+job_setup.prescript_3 = printf '#include "JobConfig/reco/CORSIKA-cosmic-general-mix.fcl"\n' >> template.fcl
+job_setup.prescript_4 = ${_CONDOR_JOB_IWD}/Code/Offline/listFiles.sh %(dataset)s > inputs.txt
+job_setup.prescript_5 = generate_fcl --desc=corsika_reco_lo --dsconf=%(date)s --inputs=inputs.txt --merge=10 --embed template.fcl
+job_setup.prescript_6 = sed -i "s/MU2EGRIDDSOWNER/%(account)s/g" 000/*.fcl
+job_setup.prescript_7 = sed -i "s/MU2EGRIDDSCONF/%(date)s/g" 000/*.fcl
+job_setup.prescript_8 = mv 000/* .
+job_setup.prescript_9 = rm template.fcl
+job_setup.ifdh_art = False
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_reco_lo_fcl_%(date)s
+executable.name = true
+
+[stage_reco_lo]
+# _poms_analysis is a keyword for POMS to instruct it to make its own dataset
+# and followup with it to propagate it as input of next stage
+job_output.add_to_dataset = _poms_analysis
+job_output.dataset_exclude = *truncated*
+job_output_2.add_to_dataset = %(account)_reco_lo_root
+global.outdir = /pnfs/mu2e/scratch/users/%(account)s/workflow/%(project_name)s_reco_lo_%(date)s
+job_setup.getconfig = True
+submit.n_files_per_job = 1
+sam_consumer.limit = 1
+submit.dataset = %(dataset)s
+executable.arg_1 = --sam-data-tier=Output:mcs
+job_setup.multifile = False
+job_setup.setup_local = True
+
+[executable]
+name = mu2e