CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
configTemplates.py
Go to the documentation of this file.
1 from alternateValidationTemplates import *
2 from offlineValidationTemplates import *
3 from geometryComparisonTemplates import *
4 from monteCarloValidationTemplates import *
5 from trackSplittingValidationTemplates import *
6 from zMuMuValidationTemplates import *
7 from TkAlExceptions import AllInOneError
8 
9 
10 ######################################################################
11 ######################################################################
12 ### ###
13 ### General Templates ###
14 ### ###
15 ######################################################################
16 ######################################################################
17 
18 ######################################################################
19 ######################################################################
20 conditionsTemplate="""
21 process.conditionsIn.oO[rcdName]Oo. = CalibTracker.Configuration.Common.PoolDBESSource_cfi.poolDBESSource.clone(
22  connect = cms.string('.oO[connectString]Oo.'),
23  toGet = cms.VPSet(cms.PSet(record = cms.string('.oO[rcdName]Oo.'),
24  tag = cms.string('.oO[tagName]Oo.')
25  )
26  )
27  )
28 process.prefer_conditionsIn.oO[rcdName]Oo. = cms.ESPrefer("PoolDBESSource", "conditionsIn.oO[rcdName]Oo.")
29 """
30 
31 
32 ######################################################################
33 ######################################################################
34 #batch job execution
35 scriptTemplate="""
36 #!/bin/bash
37 CWD=`pwd -P`
38 source /afs/cern.ch/cms/caf/setup.sh
39 cd .oO[CMSSW_BASE]Oo./src
40 export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo.
41 eval `scramv1 ru -sh`
42 # rfmkdir -p .oO[workdir]Oo.
43 # rfmkdir -p .oO[datadir]Oo.
44 
45 if [[ $HOSTNAME = lxplus[0-9]*\.cern\.ch ]] # check for interactive mode
46 then
47  rfmkdir -p .oO[workdir]Oo.
48  rm -f .oO[workdir]Oo./*
49  cd .oO[workdir]Oo.
50 else
51  mkdir -p $CWD/TkAllInOneTool
52  cd $CWD/TkAllInOneTool
53 fi
54 
55 # rm -f .oO[workdir]Oo./*
56 # cd .oO[workdir]Oo.
57 
58 #run
59 pwd
60 df -h .
61 .oO[CommandLine]Oo.
62 echo "----"
63 echo "List of files in $(pwd):"
64 ls -ltr
65 echo "----"
66 echo ""
67 
68 
69 #retrieve
70 rfmkdir -p .oO[logdir]Oo. >&! /dev/null
71 gzip -f LOGFILE_*_.oO[name]Oo..log
72 find .oO[workdir]Oo. -maxdepth 1 -name "LOGFILE*.oO[alignmentName]Oo.*" -print | xargs -I {} bash -c "rfcp {} .oO[logdir]Oo."
73 
74 #copy root files to eos
75 cmsMkdir /store/caf/user/$USER/.oO[eosdir]Oo.
76 root_files=$(ls --color=never -d *.oO[alignmentName]Oo.*.root)
77 echo ${root_files}
78 
79 for file in ${root_files}
80 do
81  cmsStage -f ${file} /store/caf/user/$USER/.oO[eosdir]Oo.
82  echo ${file}
83 done
84 
85 #cleanup
86 if [[ $HOSTNAME = lxplus[0-9]*\.cern\.ch ]] # check for interactive mode
87 then
88  rm -rf .oO[workdir]Oo.
89 fi
90 echo "done."
91 """
92 
93 
94 ######################################################################
95 ######################################################################
96 #batch job execution
97 parallelScriptTemplate="""
98 #!/bin/bash
99 #init
100 #ulimit -v 3072000
101 #export STAGE_SVCCLASS=cmscafuser
102 #save path to the LSF batch working directory (/pool/lsf)
103 export LSFWORKDIR=$PWD
104 echo LSF working directory is $LSFWORKDIR
105 source /afs/cern.ch/cms/caf/setup.sh
106 # source /afs/cern.ch/cms/sw/cmsset_default.sh
107 cd .oO[CMSSW_BASE]Oo./src
108 # export SCRAM_ARCH=slc5_amd64_gcc462
109 export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo.
110 eval `scramv1 ru -sh`
111 #rfmkdir -p ${LSFWORKDIR}
112 
113 # make rfmkdir silent in case directory already exists
114 rfmkdir -p .oO[datadir]Oo. >&! /dev/null
115 cmsMkdir /store/caf/user/$USER/.oO[eosdir]Oo.
116 
117 #remove possible result file from previous runs
118 previous_results=$(cmsLs -l /store/caf/user/$USER/.oO[eosdir]Oo. | awk '{print $5}')
119 for file in ${previous_results}
120 do
121  # if [ ${file} = *.oO[datadir]Oo./*.oO[alignmentName]Oo.*.root ]
122  if [ ${file} = /store/caf/user/$USER/.oO[eosdir]Oo./.oO[outputFile]Oo. ]
123  then
124  cmsStage -f ${file} ${file}.bak
125  fi
126 done
127 
128 #rm -f ${LSFWORKDIR}/*
129 cd ${LSFWORKDIR}
130 
131 #run
132 pwd
133 df -h .
134 .oO[CommandLine]Oo.
135 echo "----"
136 echo "List of files in $(pwd):"
137 ls -ltr
138 echo "----"
139 echo ""
140 
141 
142 #retrieve
143 rfmkdir -p .oO[logdir]Oo. >&! /dev/null
144 gzip LOGFILE_*_.oO[name]Oo..log
145 find ${LSFWORKDIR} -maxdepth 1 -name "LOGFILE*.oO[alignmentName]Oo.*" -print | xargs -I {} bash -c "rfcp {} .oO[logdir]Oo."
146 
147 #copy root files to eos
148 cmsMkdir /store/caf/user/$USER/.oO[eosdir]Oo.
149 root_files=$(ls --color=never -d ${LSFWORKDIR}/*.oO[alignmentName]Oo._.oO[nIndex]Oo.*.root)
150 echo "ls"
151 ls
152 echo "\${root_files}:"
153 echo ${root_files}
154 for file in ${root_files}
155 do
156  # echo "cmsStage -f ${file} /store/caf/user/$USER/.oO[eosdir]Oo."
157  cmsStage -f ${file} /store/caf/user/$USER/.oO[eosdir]Oo.
158  # echo ${file}
159 done
160 
161 #cleanup - do not remove workdir, since another parallel job might be running in the same node
162 find ${LSFWORKDIR} -maxdepth 1 -name "*.oO[alignmentName]Oo._.oO[nIndex]Oo.*.root" -print | xargs -I {} bash -c "rm {}"
163 echo "done."
164 """
165 
166 
167 ######################################################################
168 ######################################################################
169 mergeTemplate="""
170 #!/bin/bash
171 CWD=`pwd -P`
172 cd .oO[CMSSW_BASE]Oo./src
173 export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo.
174 eval `scramv1 ru -sh`
175 
176 #create results-directory and copy used configuration there
177 rfmkdir -p .oO[datadir]Oo.
178 rfcp .oO[logdir]Oo./usedConfiguration.ini .oO[datadir]Oo.
179 
180 if [[ $HOSTNAME = lxplus[0-9]*\.cern\.ch ]] # check for interactive mode
181 then
182  mkdir -p .oO[workdir]Oo.
183  cd .oO[workdir]Oo.
184 else
185  cd $CWD
186 fi
187 echo "Working directory: $(pwd -P)"
188 
189 ###############################################################################
190 # download root files from eos
191 root_files=$(cmsLs -l /store/caf/user/$USER/.oO[eosdir]Oo. | awk '{print $5}' \
192  | grep ".root$" | grep -v "result.root$")
193 for file in ${root_files}
194 do
195  cmsStage -f ${file} .
196  # echo ${file}
197 done
198 
199 
200 #run
201 .oO[DownloadData]Oo.
202 .oO[CompareAlignments]Oo.
203 
204 .oO[RunExtendedOfflineValidation]Oo.
205 .oO[RunTrackSplitPlot]Oo.
206 
207 # clean-up
208 # ls -l *.root
209 rm -f *.root
210 
211 #zip stdout and stderr from the farm jobs
212 cd .oO[logdir]Oo.
213 find . -name "*.stderr" -exec gzip -f {} \;
214 find . -name "*.stdout" -exec gzip -f {} \;
215 """
216 
217 
218 ######################################################################
219 ######################################################################
220 compareAlignmentsExecution="""
221 #merge for .oO[validationId]Oo. if it does not exist or is not up-to-date
222 echo -e "\n\nComparing validations"
223 cp .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/scripts/compareFileAges.C .
224 root -x -q -b -l "compareFileAges.C(\\\"root://eoscms.cern.ch//eos/cms/store/caf/user/$USER/.oO[eosdir]Oo./.oO[validationId]Oo._result.root\\\", \\\".oO[compareStringsPlain]Oo.\\\")"
225 comparisonNeeded=${?}
226 
227 if [[ ${comparisonNeeded} -eq 1 ]]
228 then
229  cp .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/scripts/compareAlignments.cc .
230  root -x -q -b -l 'compareAlignments.cc++(\".oO[compareStrings]Oo.\")'
231  mv result.root .oO[validationId]Oo._result.root
232  cmsStage -f .oO[validationId]Oo._result.root /store/caf/user/$USER/.oO[eosdir]Oo.
233 else
234  echo ".oO[validationId]Oo._result.root is up-to-date, no need to compare again."
235  cmsStage -f /store/caf/user/$USER/.oO[eosdir]Oo./.oO[validationId]Oo._result.root .
236 fi
237 """
238 
239 
240 ######################################################################
241 ######################################################################
242 extendedValidationExecution="""
243 #run extended offline validation scripts
244 echo -e "\n\nRunning extended offline validation"
245 if [[ $HOSTNAME = lxplus[0-9]*\.cern\.ch ]] # check for interactive mode
246 then
247  rfmkdir -p .oO[workdir]Oo./ExtendedOfflineValidation_Images
248 else
249  mkdir -p ExtendedOfflineValidation_Images
250 fi
251 
252 rfcp .oO[extendeValScriptPath]Oo. .
253 rfcp .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/macros/PlotAlignmentValidation.C .
254 root -x -b -q -l TkAlExtendedOfflineValidation.C
255 rfmkdir -p .oO[datadir]Oo./ExtendedOfflineValidation_Images
256 
257 if [[ $HOSTNAME = lxplus[0-9]*\.cern\.ch ]] # check for interactive mode
258 then
259  image_files=$(ls --color=never | find .oO[workdir]Oo./ExtendedOfflineValidation_Images/ -name \*ps -o -name \*root)
260  echo -e "\n\nProduced plot files:"
261  #echo ${image_files}
262  ls .oO[workdir]Oo./ExtendedOfflineValidation_Images
263 else
264  image_files=$(ls --color=never | find ExtendedOfflineValidation_Images/ -name \*ps -o -name \*root)
265  echo -e "\n\nProduced plot files:"
266  #echo ${image_files}
267  ls ExtendedOfflineValidation_Images
268 fi
269 
270 for image in ${image_files}
271 do
272  cp ${image} .oO[datadir]Oo./ExtendedOfflineValidation_Images
273 done
274 """
275 
276 
277 ######################################################################
278 ######################################################################
279 extendedValidationTemplate="""
280 void TkAlExtendedOfflineValidation()
281 {
282  // load framework lite just to find the CMSSW libs...
283  gSystem->Load("libFWCoreFWLite");
284  AutoLibraryLoader::enable();
285  //compile the makro
286  gROOT->ProcessLine(".L .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/macros/PlotAlignmentValidation.C++");
287  // gROOT->ProcessLine(".L ./PlotAlignmentValidation.C++");
288 
289  .oO[extendedInstantiation]Oo.
290  p.setOutputDir("./ExtendedOfflineValidation_Images");
291  p.setTreeBaseDir(".oO[OfflineTreeBaseDir]Oo.");
292  p.plotDMR(".oO[DMRMethod]Oo.",.oO[DMRMinimum]Oo.,".oO[DMROptions]Oo.");
293  p.plotSurfaceShapes(".oO[SurfaceShapes]Oo.");
294  p.plotChi2(".oO[resultPlotFile]Oo._result.root");
295 }
296 """
297 
298 
299 ######################################################################
300 ######################################################################
301 crabCfgTemplate="""
302 [CRAB]
303 jobtype = cmssw
304 scheduler = caf
305 use_server = 0
306 
307 [CMSSW]
308 datasetpath = .oO[dataset]Oo.
309 pset = .oO[cfgFile]Oo.
310 total_number_of_.oO[McOrData]Oo.
311 number_of_jobs = .oO[numberOfJobs]Oo.
312 output_file = .oO[outputFile]Oo.
313 runselection = .oO[runRange]Oo.
314 lumi_mask = .oO[JSON]Oo.
315 
316 [USER]
317 return_data = 0
318 copy_data = 1
319 storage_element = T2_CH_CERN
320 user_remote_dir = .oO[eosdir]Oo.
321 ui_working_dir = .oO[crabWorkingDir]Oo.
322 # script_exe = .oO[script]Oo.
323 # .oO[email]Oo.
324 
325 [CAF]
326 queue = .oO[queue]Oo.
327 """
328 
329 
330 
331 
332 ######################################################################
333 ######################################################################
334 ### ###
335 ### Alternate Templates ###
336 ### ###
337 ######################################################################
338 ######################################################################
339 
340 
341 def alternateTemplate( templateName, alternateTemplateName ):
342 
343  if not templateName in globals().keys():
344  msg = "unkown template to replace %s"%templateName
345  raise AllInOneError(msg)
346  if not alternateTemplateName in globals().keys():
347  msg = "unkown template to replace %s"%alternateTemplateName
348  raise AllInOneError(msg)
349  globals()[ templateName ] = globals()[ alternateTemplateName ]
350  # = eval("configTemplates.%s"%"alternateTemplate")
configTemplates.alternateTemplate
def alternateTemplate
### Alternate Templates ###
Definition: configTemplates.py:341
Generated for CMSSW Reference Manual by doxygen 1.8.5