CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
createJobs.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 
3 import os, sys, optparse, math
4 
5 copyargs = sys.argv[:]
6 for i in range(len(copyargs)):
7  if copyargs[i] == "":
8  copyargs[i] = "\"\""
9  if copyargs[i].find(" ") != -1:
10  copyargs[i] = "\"%s\"" % copyargs[i]
11 commandline = " ".join(copyargs)
12 
13 prog = sys.argv[0]
14 
15 usage = """./%(prog)s DIRNAME ITERATIONS INITIALGEOM INPUTFILES [options]
16 
17 Creates (overwrites) a directory for each of the iterations and creates (overwrites)
18 submitJobs.sh with the submission sequence and dependencies.
19 
20 DIRNAME directories will be named DIRNAME01, DIRNAME02, etc.
21 ITERATIONS number of iterations
22 INITIALGEOM SQLite file containing muon geometry with tag names
23  DTAlignmentRcd, DTAlignmentErrorExtendedRcd, CSCAlignmentRcd, CSCAlignmentErrorExtendedRcd
24 INPUTFILES Python file defining 'fileNames', a list of input files as
25  strings (create with findQualityFiles.py)""" % vars()
26 
27 parser = optparse.OptionParser(usage)
28 parser.add_option("-j", "--jobs",
29  help="approximate number of \"gather\" subjobs",
30  type="int",
31  default=50,
32  dest="subjobs")
33 parser.add_option("-s", "--submitJobs",
34  help="alternate name of submitJobs.sh script (please include .sh extension); a file with this name will be OVERWRITTEN",
35  type="string",
36  default="submitJobs.sh",
37  dest="submitJobs")
38 parser.add_option("-b", "--big",
39  help="if invoked, subjobs will also be run on cmscaf1nd",
40  action="store_true",
41  dest="big")
42 parser.add_option("-u", "--user_mail",
43  help="if invoked, send mail to a specified email destination. If \"-u\" is not present, the default destination LSB_MAILTO in lsf.conf will be used",
44  type="string",
45  dest="user_mail")
46 parser.add_option("--mapplots",
47  help="if invoked, draw \"map plots\"",
48  action="store_true",
49  dest="mapplots")
50 parser.add_option("--segdiffplots",
51  help="if invoked, draw \"segment-difference plots\"",
52  action="store_true",
53  dest="segdiffplots")
54 parser.add_option("--curvatureplots",
55  help="if invoked, draw \"curvature plots\"",
56  action="store_true",
57  dest="curvatureplots")
58 parser.add_option("--globalTag",
59  help="GlobalTag for alignment/calibration conditions (typically all conditions except muon and tracker alignment)",
60  type="string",
61  default="CRAFT0831X_V1::All",
62  dest="globaltag")
63 parser.add_option("--trackerconnect",
64  help="connect string for tracker alignment (frontier://FrontierProd/CMS_COND_310X_ALIGN or sqlite_file:...)",
65  type="string",
66  default="",
67  dest="trackerconnect")
68 parser.add_option("--trackeralignment",
69  help="name of TrackerAlignmentRcd tag",
70  type="string",
71  default="Alignments",
72  dest="trackeralignment")
73 parser.add_option("--trackerAPEconnect",
74  help="connect string for tracker APEs (frontier://... or sqlite_file:...)",
75  type="string",
76  default="",
77  dest="trackerAPEconnect")
78 parser.add_option("--trackerAPE",
79  help="name of TrackerAlignmentErrorExtendedRcd tag (tracker APEs)",
80  type="string",
81  default="AlignmentErrorsExtended",
82  dest="trackerAPE")
83 parser.add_option("--trackerBowsconnect",
84  help="connect string for tracker Surface Deformations (frontier://... or sqlite_file:...)",
85  type="string",
86  default="",
87  dest="trackerBowsconnect")
88 parser.add_option("--trackerBows",
89  help="name of TrackerSurfaceDeformationRcd tag",
90  type="string",
91  default="TrackerSurfaceDeformations",
92  dest="trackerBows")
93 parser.add_option("--gprcdconnect",
94  help="connect string for GlobalPositionRcd (frontier://... or sqlite_file:...)",
95  type="string",
96  default="",
97  dest="gprcdconnect")
98 parser.add_option("--gprcd",
99  help="name of GlobalPositionRcd tag",
100  type="string",
101  default="GlobalPosition",
102  dest="gprcd")
103 parser.add_option("--iscosmics",
104  help="if invoked, use cosmic track refitter instead of the standard one",
105  action="store_true",
106  dest="iscosmics")
107 parser.add_option("--station123params",
108  help="alignable parameters for DT stations 1, 2, 3 (see SWGuideAlignmentAlgorithms#Selection_of_what_to_align)",
109  type="string",
110  default="111111",
111  dest="station123params")
112 parser.add_option("--station4params",
113  help="alignable parameters for DT station 4",
114  type="string",
115  default="100011",
116  dest="station4params")
117 parser.add_option("--cscparams",
118  help="alignable parameters for CSC chambers",
119  type="string",
120  default="100011",
121  dest="cscparams")
122 parser.add_option("--minTrackPt",
123  help="minimum allowed track transverse momentum (in GeV)",
124  type="string",
125  default="0",
126  dest="minTrackPt")
127 parser.add_option("--maxTrackPt",
128  help="maximum allowed track transverse momentum (in GeV)",
129  type="string",
130  default="1000",
131  dest="maxTrackPt")
132 parser.add_option("--minTrackP",
133  help="minimum allowed track momentum (in GeV)",
134  type="string",
135  default="0",
136  dest="minTrackP")
137 parser.add_option("--maxTrackP",
138  help="maximum allowed track momentum (in GeV)",
139  type="string",
140  default="10000",
141  dest="maxTrackP")
142 parser.add_option("--minTrackerHits",
143  help="minimum number of tracker hits",
144  type="int",
145  default=15,
146  dest="minTrackerHits")
147 parser.add_option("--maxTrackerRedChi2",
148  help="maximum tracker chi^2 per degrees of freedom",
149  type="string",
150  default="10",
151  dest="maxTrackerRedChi2")
152 parser.add_option("--notAllowTIDTEC",
153  help="if invoked, do not allow tracks that pass through the tracker's TID||TEC region (not recommended)",
154  action="store_true",
155  dest="notAllowTIDTEC")
156 parser.add_option("--twoBin",
157  help="if invoked, apply the \"two-bin method\" to control charge-antisymmetric errors",
158  action="store_true",
159  dest="twoBin")
160 parser.add_option("--weightAlignment",
161  help="if invoked, segments will be weighted by ndf/chi^2 in the alignment",
162  action="store_true",
163  dest="weightAlignment")
164 parser.add_option("--minAlignmentSegments",
165  help="minimum number of segments required to align a chamber",
166  type="int",
167  default=5,
168  dest="minAlignmentHits")
169 parser.add_option("--notCombineME11",
170  help="if invoced, treat ME1/1a and ME1/1b as separate objects",
171  action="store_true",
172  dest="notCombineME11")
173 parser.add_option("--maxEvents",
174  help="maximum number of events",
175  type="string",
176  default="-1",
177  dest="maxEvents")
178 parser.add_option("--skipEvents",
179  help="number of events to be skipped",
180  type="string",
181  default="0",
182  dest="skipEvents")
183 parser.add_option("--validationLabel",
184  help="if given nonempty string RUNLABEL, diagnostics and creation of plots will be run in the end of the last iteration; the RUNLABEL will be used to mark a run; the results will be put into a RUNLABEL_DATESTAMP.tgz tarball",
185  type="string",
186  default="",
187  dest="validationLabel")
188 parser.add_option("--maxResSlopeY",
189  help="maximum residual slope y component",
190  type="string",
191  default="10",
192  dest="maxResSlopeY")
193 parser.add_option("--motionPolicyNSigma",
194  help="minimum nsigma(deltax) position displacement in order to move a chamber for the final alignment result; default NSIGMA=3",
195  type="int",
196  default=3,
197  dest="motionPolicyNSigma")
198 parser.add_option("--noCleanUp",
199  help="if invoked, temporary plotting???.root and *.tmp files would not be removed at the end of each align job",
200  action="store_true",
201  dest="noCleanUp")
202 parser.add_option("--noCSC",
203  help="if invoked, CSC endcap chambers would not be processed",
204  action="store_true",
205  dest="noCSC")
206 parser.add_option("--noDT",
207  help="if invoked, DT barrel chambers would not be processed",
208  action="store_true",
209  dest="noDT")
210 parser.add_option("--createMapNtuple",
211  help="if invoked while mapplots are switched on, a special ntuple would be created",
212  action="store_true",
213  dest="createMapNtuple")
214 parser.add_option("--inputInBlocks",
215  help="if invoked, assume that INPUTFILES provides a list of files already groupped into job blocks, -j has no effect in that case",
216  action="store_true",
217  dest="inputInBlocks")
218 parser.add_option("--json",
219  help="If present with JSON file as argument, use JSON file for good lumi mask. "+\
220  "The latest JSON file is available at /afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions11/7TeV/Prompt/",
221  type="string",
222  default="",
223  dest="json")
224 parser.add_option("--createAlignNtuple",
225  help="if invoked, debug ntuples with residuals would be created during gather jobs",
226  action="store_true",
227  dest="createAlignNtuple")
228 parser.add_option("--residualsModel",
229  help="functional residuals model. Possible vaslues: pureGaussian2D (default), pureGaussian, GaussPowerTails, ROOTVoigt, powerLawTails",
230  type="string",
231  default="pureGaussian2D",
232  dest="residualsModel")
233 parser.add_option("--useResiduals",
234  help="select residuals to use, possible values: 1111, 1110, 1100, 1010, 0010 that correspond to x y dxdz dydz residuals",
235  type="string",
236  default="1110",
237  dest="useResiduals")
238 parser.add_option("--peakNSigma",
239  help="if >0, only residuals peaks within n-sigma multidimentional ellipsoid would be considered in the alignment fit",
240  type="string",
241  default="-1.",
242  dest="peakNSigma")
243 parser.add_option("--preFilter",
244  help="if invoked, MuonAlignmentPreFilter module would be invoked in the Path's beginning. Can significantly speed up gather jobs.",
245  action="store_true",
246  dest="preFilter")
247 parser.add_option("--muonCollectionTag",
248  help="If empty, use trajectories. If not empty, it's InputTag of muons collection to use in tracker muons based approach, e.g., 'newmuons' or 'muons'",
249  type="string",
250  default="",
251  dest="muonCollectionTag")
252 parser.add_option("--maxDxy",
253  help="maximum track impact parameter with relation to beamline",
254  type="string",
255  default="1000.",
256  dest="maxDxy")
257 parser.add_option("--minNCrossedChambers",
258  help="minimum number of muon chambers that a track is required to cross",
259  type="string",
260  default="3",
261  dest="minNCrossedChambers")
262 parser.add_option("--extraPlots",
263  help="produce additional plots with geometry, reports differences, and corrections visulizations",
264  action="store_true",
265  dest="extraPlots")
266 
267 if len(sys.argv) < 5:
268  raise SystemError("Too few arguments.\n\n"+parser.format_help())
269 
270 DIRNAME = sys.argv[1]
271 ITERATIONS = int(sys.argv[2])
272 INITIALGEOM = sys.argv[3]
273 INPUTFILES = sys.argv[4]
274 
275 options, args = parser.parse_args(sys.argv[5:])
276 user_mail = options.user_mail
277 mapplots_ingeneral = options.mapplots
278 segdiffplots_ingeneral = options.segdiffplots
279 curvatureplots_ingeneral = options.curvatureplots
280 globaltag = options.globaltag
281 trackerconnect = options.trackerconnect
282 trackeralignment = options.trackeralignment
283 trackerAPEconnect = options.trackerAPEconnect
284 trackerAPE = options.trackerAPE
285 trackerBowsconnect = options.trackerBowsconnect
286 trackerBows = options.trackerBows
287 gprcdconnect = options.gprcdconnect
288 gprcd = options.gprcd
289 iscosmics = str(options.iscosmics)
290 station123params = options.station123params
291 station4params = options.station4params
292 cscparams = options.cscparams
293 muonCollectionTag = options.muonCollectionTag
294 minTrackPt = options.minTrackPt
295 maxTrackPt = options.maxTrackPt
296 minTrackP = options.minTrackP
297 maxTrackP = options.maxTrackP
298 maxDxy = options.maxDxy
299 minTrackerHits = str(options.minTrackerHits)
300 maxTrackerRedChi2 = options.maxTrackerRedChi2
301 minNCrossedChambers = options.minNCrossedChambers
302 allowTIDTEC = str(not options.notAllowTIDTEC)
303 twoBin = str(options.twoBin)
304 weightAlignment = str(options.weightAlignment)
305 minAlignmentHits = str(options.minAlignmentHits)
306 combineME11 = str(not options.notCombineME11)
307 maxEvents = options.maxEvents
308 skipEvents = options.skipEvents
309 validationLabel = options.validationLabel
310 maxResSlopeY = options.maxResSlopeY
311 theNSigma = options.motionPolicyNSigma
312 residualsModel = options.residualsModel
313 peakNSigma = options.peakNSigma
314 preFilter = not not options.preFilter
315 extraPlots = options.extraPlots
316 useResiduals = options.useResiduals
317 
318 
319 #print "check: ", allowTIDTEC, combineME11, preFilter
320 
321 doCleanUp = not options.noCleanUp
322 createMapNtuple = not not options.createMapNtuple
323 createAlignNtuple = not not options.createAlignNtuple
324 
325 doCSC = True
326 if options.noCSC: doCSC = False
327 doDT = True
328 if options.noDT: doDT = False
329 if options.noCSC and options.noDT:
330  print "cannot do --noCSC and --noDT at the same time!"
331  sys.exit()
332 
333 json_file = options.json
334 
335 fileNames=[]
336 fileNamesBlocks=[]
337 execfile(INPUTFILES)
338 njobs = options.subjobs
339 if (options.inputInBlocks):
340  njobs = len(fileNamesBlocks)
341  if njobs==0:
342  print "while --inputInBlocks is specified, the INPUTFILES has no blocks!"
343  sys.exit()
344 
345 stepsize = int(math.ceil(1.*len(fileNames)/options.subjobs))
346 
347 pwd = str(os.getcwdu())
348 
349 copytrackerdb = ""
350 if trackerconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerconnect[12:]
351 if trackerAPEconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerAPEconnect[12:]
352 if trackerBowsconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerBowsconnect[12:]
353 if gprcdconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % gprcdconnect[12:]
354 
355 
356 #####################################################################
357 # step 0: convert initial geometry to xml
358 INITIALXML = INITIALGEOM + '.xml'
359 if INITIALGEOM[-3:]=='.db':
360  INITIALXML = INITIALGEOM[:-3] + '.xml'
361 print "Converting",INITIALGEOM,"to",INITIALXML," ...will be done in several seconds..."
362 print "./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %s %s --gprcdconnect %s --gprcd %s" % (INITIALGEOM,INITIALXML,gprcdconnect,gprcd)
363 exit_code = os.system("./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %s %s --gprcdconnect %s --gprcd %s" % (INITIALGEOM,INITIALXML,gprcdconnect,gprcd))
364 if exit_code>0:
365  print "problem: conversion exited with code:", exit_code
366  sys.exit()
367 
368 #####################################################################
369 
370 def writeGatherCfg(fname, my_vars):
371  file(fname, "w").write("""#/bin/sh
372 # %(commandline)s
373 
374 export ALIGNMENT_CAFDIR=`pwd`
375 
376 cd %(pwd)s
377 eval `scramv1 run -sh`
378 export ALIGNMENT_AFSDIR=`pwd`
379 
380 export ALIGNMENT_INPUTFILES='%(inputfiles)s'
381 export ALIGNMENT_ITERATION=%(iteration)d
382 export ALIGNMENT_JOBNUMBER=%(jobnumber)d
383 export ALIGNMENT_MAPPLOTS=%(mapplots)s
384 export ALIGNMENT_SEGDIFFPLOTS=%(segdiffplots)s
385 export ALIGNMENT_CURVATUREPLOTS=%(curvatureplots)s
386 export ALIGNMENT_GLOBALTAG=%(globaltag)s
387 export ALIGNMENT_INPUTDB=%(inputdb)s
388 export ALIGNMENT_TRACKERCONNECT=%(trackerconnect)s
389 export ALIGNMENT_TRACKERALIGNMENT=%(trackeralignment)s
390 export ALIGNMENT_TRACKERAPECONNECT=%(trackerAPEconnect)s
391 export ALIGNMENT_TRACKERAPE=%(trackerAPE)s
392 export ALIGNMENT_TRACKERBOWSCONNECT=%(trackerBowsconnect)s
393 export ALIGNMENT_TRACKERBOWS=%(trackerBows)s
394 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
395 export ALIGNMENT_GPRCD=%(gprcd)s
396 export ALIGNMENT_ISCOSMICS=%(iscosmics)s
397 export ALIGNMENT_STATION123PARAMS=%(station123params)s
398 export ALIGNMENT_STATION4PARAMS=%(station4params)s
399 export ALIGNMENT_CSCPARAMS=%(cscparams)s
400 export ALIGNMENT_MUONCOLLECTIONTAG=%(muonCollectionTag)s
401 export ALIGNMENT_MINTRACKPT=%(minTrackPt)s
402 export ALIGNMENT_MAXTRACKPT=%(maxTrackPt)s
403 export ALIGNMENT_MINTRACKP=%(minTrackP)s
404 export ALIGNMENT_MAXTRACKP=%(maxTrackP)s
405 export ALIGNMENT_MAXDXY=%(maxDxy)s
406 export ALIGNMENT_MINTRACKERHITS=%(minTrackerHits)s
407 export ALIGNMENT_MAXTRACKERREDCHI2=%(maxTrackerRedChi2)s
408 export ALIGNMENT_MINNCROSSEDCHAMBERS=%(minNCrossedChambers)s
409 export ALIGNMENT_ALLOWTIDTEC=%(allowTIDTEC)s
410 export ALIGNMENT_TWOBIN=%(twoBin)s
411 export ALIGNMENT_WEIGHTALIGNMENT=%(weightAlignment)s
412 export ALIGNMENT_MINALIGNMENTHITS=%(minAlignmentHits)s
413 export ALIGNMENT_COMBINEME11=%(combineME11)s
414 export ALIGNMENT_MAXEVENTS=%(maxEvents)s
415 export ALIGNMENT_SKIPEVENTS=%(skipEvents)s
416 export ALIGNMENT_MAXRESSLOPEY=%(maxResSlopeY)s
417 export ALIGNMENT_DO_DT=%(doDT)s
418 export ALIGNMENT_DO_CSC=%(doCSC)s
419 export ALIGNMENT_JSON=%(json_file)s
420 export ALIGNMENT_CREATEMAPNTUPLE=%(createMapNtuple)s
421 #export ALIGNMENT_CREATEALIGNNTUPLE=%(createAlignNtuple)s
422 export ALIGNMENT_PREFILTER=%(preFilter)s
423 
424 
425 if [ \"zzz$ALIGNMENT_JSON\" != \"zzz\" ]; then
426  cp -f $ALIGNMENT_JSON $ALIGNMENT_CAFDIR/
427 fi
428 
429 cp -f %(directory)sgather_cfg.py %(inputdbdir)s%(inputdb)s %(copytrackerdb)s $ALIGNMENT_CAFDIR/
430 cd $ALIGNMENT_CAFDIR/
431 ls -l
432 cmsRun gather_cfg.py
433 ls -l
434 cp -f *.tmp %(copyplots)s $ALIGNMENT_AFSDIR/%(directory)s
435 """ % my_vars)
436 
437 #####################################################################
438 
439 def writeAlignCfg(fname, my_vars):
440  file("%salign.sh" % directory, "w").write("""#!/bin/sh
441 # %(commandline)s
442 
443 export ALIGNMENT_CAFDIR=`pwd`
444 
445 cd %(pwd)s
446 eval `scramv1 run -sh`
447 export ALIGNMENT_AFSDIR=`pwd`
448 export ALIGNMENT_INPUTDB=%(inputdb)s
449 export ALIGNMENT_ITERATION=%(iteration)d
450 export ALIGNMENT_GLOBALTAG=%(globaltag)s
451 export ALIGNMENT_TRACKERCONNECT=%(trackerconnect)s
452 export ALIGNMENT_TRACKERALIGNMENT=%(trackeralignment)s
453 export ALIGNMENT_TRACKERAPECONNECT=%(trackerAPEconnect)s
454 export ALIGNMENT_TRACKERAPE=%(trackerAPE)s
455 export ALIGNMENT_TRACKERBOWSCONNECT=%(trackerBowsconnect)s
456 export ALIGNMENT_TRACKERBOWS=%(trackerBows)s
457 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
458 export ALIGNMENT_GPRCD=%(gprcd)s
459 export ALIGNMENT_ISCOSMICS=%(iscosmics)s
460 export ALIGNMENT_STATION123PARAMS=%(station123params)s
461 export ALIGNMENT_STATION4PARAMS=%(station4params)s
462 export ALIGNMENT_CSCPARAMS=%(cscparams)s
463 export ALIGNMENT_MINTRACKPT=%(minTrackPt)s
464 export ALIGNMENT_MAXTRACKPT=%(maxTrackPt)s
465 export ALIGNMENT_MINTRACKP=%(minTrackP)s
466 export ALIGNMENT_MAXTRACKP=%(maxTrackP)s
467 export ALIGNMENT_MINTRACKERHITS=%(minTrackerHits)s
468 export ALIGNMENT_MAXTRACKERREDCHI2=%(maxTrackerRedChi2)s
469 export ALIGNMENT_ALLOWTIDTEC=%(allowTIDTEC)s
470 export ALIGNMENT_TWOBIN=%(twoBin)s
471 export ALIGNMENT_WEIGHTALIGNMENT=%(weightAlignment)s
472 export ALIGNMENT_MINALIGNMENTHITS=%(minAlignmentHits)s
473 export ALIGNMENT_COMBINEME11=%(combineME11)s
474 export ALIGNMENT_MAXRESSLOPEY=%(maxResSlopeY)s
475 export ALIGNMENT_CLEANUP=%(doCleanUp)s
476 export ALIGNMENT_CREATEALIGNNTUPLE=%(createAlignNtuple)s
477 export ALIGNMENT_RESIDUALSMODEL=%(residualsModel)s
478 export ALIGNMENT_PEAKNSIGMA=%(peakNSigma)s
479 export ALIGNMENT_USERESIDUALS=%(useResiduals)s
480 
481 cp -f %(directory)salign_cfg.py %(inputdbdir)s%(inputdb)s %(directory)s*.tmp %(copytrackerdb)s $ALIGNMENT_CAFDIR/
482 
483 export ALIGNMENT_PLOTTINGTMP=`find %(directory)splotting0*.root -maxdepth 1 -size +0 -print 2> /dev/null`
484 
485 # if it's 1st or last iteration, combine _plotting.root files into one:
486 if [ \"$ALIGNMENT_ITERATION\" != \"111\" ] || [ \"$ALIGNMENT_ITERATION\" == \"%(ITERATIONS)s\" ]; then
487  #nfiles=$(ls %(directory)splotting0*.root 2> /dev/null | wc -l)
488  if [ \"zzz$ALIGNMENT_PLOTTINGTMP\" != \"zzz\" ]; then
489  hadd -f1 %(directory)s%(director)s_plotting.root %(directory)splotting0*.root
490  #if [ $? == 0 ] && [ \"$ALIGNMENT_CLEANUP\" == \"True\" ]; then rm %(directory)splotting0*.root; fi
491  fi
492 fi
493 
494 if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ \"zzz$ALIGNMENT_PLOTTINGTMP\" != \"zzz\" ]; then
495  rm $ALIGNMENT_PLOTTINGTMP
496 fi
497 
498 cd $ALIGNMENT_CAFDIR/
499 export ALIGNMENT_ALIGNMENTTMP=`find alignment*.tmp -maxdepth 1 -size +1k -print 2> /dev/null`
500 ls -l
501 
502 cmsRun align_cfg.py
503 cp -f MuonAlignmentFromReference_report.py $ALIGNMENT_AFSDIR/%(directory)s%(director)s_report.py
504 cp -f MuonAlignmentFromReference_outputdb.db $ALIGNMENT_AFSDIR/%(directory)s%(director)s.db
505 cp -f MuonAlignmentFromReference_plotting.root $ALIGNMENT_AFSDIR/%(directory)s%(director)s.root
506 
507 cd $ALIGNMENT_AFSDIR
508 ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s.db %(directory)s%(director)s.xml --noLayers --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
509 
510 export ALIGNMENT_ALIGNMENTTMP=`find %(directory)salignment*.tmp -maxdepth 1 -size +1k -print 2> /dev/null`
511 if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ \"zzz$ALIGNMENT_ALIGNMENTTMP\" != \"zzz\" ]; then
512  rm $ALIGNMENT_ALIGNMENTTMP
513  echo " "
514 fi
515 
516 # if it's not 1st or last iteration, do some clean up:
517 if [ \"$ALIGNMENT_ITERATION\" != \"1\" ] && [ \"$ALIGNMENT_ITERATION\" != \"%(ITERATIONS)s\" ]; then
518  if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ -e %(directory)s%(director)s.root ]; then
519  rm %(directory)s%(director)s.root
520  fi
521 fi
522 
523 # if it's last iteration, apply chamber motion policy
524 if [ \"$ALIGNMENT_ITERATION\" == \"%(ITERATIONS)s\" ]; then
525  # convert this iteration's geometry into detailed xml
526  ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s.db %(directory)s%(director)s_extra.xml --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
527  # perform motion policy
528  ./Alignment/MuonAlignmentAlgorithms/scripts/motionPolicyChamber.py \
529  %(INITIALXML)s %(directory)s%(director)s_extra.xml \
530  %(directory)s%(director)s_report.py \
531  %(directory)s%(director)s_final.xml \
532  --nsigma %(theNSigma)s
533  # convert the resulting xml into the final sqlite geometry
534  ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s_final.xml %(directory)s%(director)s_final.db --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
535 fi
536 
537 """ % my_vars)
538 
539 #####################################################################
540 
541 def writeValidationCfg(fname, my_vars):
542  file(fname, "w").write("""#!/bin/sh
543 # %(commandline)s
544 
545 export ALIGNMENT_CAFDIR=`pwd`
546 mkdir files
547 mkdir out
548 
549 cd %(pwd)s
550 eval `scramv1 run -sh`
551 ALIGNMENT_AFSDIR=`pwd`
552 ALIGNMENT_ITERATION=%(iteration)d
553 ALIGNMENT_MAPPLOTS=None
554 ALIGNMENT_SEGDIFFPLOTS=None
555 ALIGNMENT_CURVATUREPLOTS=None
556 ALIGNMENT_EXTRAPLOTS=%(extraPlots)s
557 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
558 export ALIGNMENT_GPRCD=%(gprcd)s
559 export ALIGNMENT_DO_DT=%(doDT)s
560 export ALIGNMENT_DO_CSC=%(doCSC)s
561 
562 
563 # copy the scripts to CAFDIR
564 cd Alignment/MuonAlignmentAlgorithms/scripts/
565 cp -f plotscripts.py $ALIGNMENT_CAFDIR/
566 cp -f mutypes.py $ALIGNMENT_CAFDIR/
567 cp -f alignmentValidation.py $ALIGNMENT_CAFDIR/
568 cp -f phiedges_fitfunctions.C $ALIGNMENT_CAFDIR/
569 cp -f createTree.py $ALIGNMENT_CAFDIR/
570 cp -f signConventions.py $ALIGNMENT_CAFDIR/
571 cp -f convertSQLiteXML.py $ALIGNMENT_CAFDIR/
572 cp -f wrapperExtraPlots.sh $ALIGNMENT_CAFDIR/
573 cd -
574 cp Alignment/MuonAlignmentAlgorithms/test/browser/tree* $ALIGNMENT_CAFDIR/out/
575 
576 # copy the results to CAFDIR
577 cp -f %(directory1)s%(director1)s_report.py $ALIGNMENT_CAFDIR/files/
578 cp -f %(directory)s%(director)s_report.py $ALIGNMENT_CAFDIR/files/
579 cp -f %(directory1)s%(director1)s.root $ALIGNMENT_CAFDIR/files/
580 cp -f %(directory)s%(director)s.root $ALIGNMENT_CAFDIR/files/
581 if [ -e %(directory1)s%(director1)s_plotting.root ] && [ -e %(directory)s%(director)s_plotting.root ]; then
582  cp -f %(directory1)s%(director1)s_plotting.root $ALIGNMENT_CAFDIR/files/
583  cp -f %(directory)s%(director)s_plotting.root $ALIGNMENT_CAFDIR/files/
584  ALIGNMENT_MAPPLOTS=%(mapplots)s
585  ALIGNMENT_SEGDIFFPLOTS=%(segdiffplots)s
586  ALIGNMENT_CURVATUREPLOTS=%(curvatureplots)s
587 fi
588 
589 dtcsc=""
590 if [ $ALIGNMENT_DO_DT == \"True\" ]; then
591  dtcsc="--dt"
592 fi
593 if [ $ALIGNMENT_DO_CSC == \"True\" ]; then
594  dtcsc="${dtcsc} --csc"
595 fi
596 
597 
598 cd $ALIGNMENT_CAFDIR/
599 echo \" ### Start running ###\"
600 date
601 
602 # do fits and median plots first
603 ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out --createDirSructure --dt --csc --fit --median
604 
605 if [ $ALIGNMENT_MAPPLOTS == \"True\" ]; then
606  ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out $dtcsc --map
607 fi
608 
609 if [ $ALIGNMENT_SEGDIFFPLOTS == \"True\" ]; then
610  ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out $dtcsc --segdiff
611 fi
612 
613 if [ $ALIGNMENT_CURVATUREPLOTS == \"True\" ]; then
614  ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out $dtcsc --curvature
615 fi
616 
617 if [ $ALIGNMENT_EXTRAPLOTS == \"True\" ]; then
618  if [ \"zzz%(copytrackerdb)s\" != \"zzz\" ]; then
619  cp -f $ALIGNMENT_AFSDIR/%(copytrackerdb)s $ALIGNMENT_CAFDIR/
620  fi
621  cp $ALIGNMENT_AFSDIR/inertGlobalPositionRcd.db .
622  ./convertSQLiteXML.py $ALIGNMENT_AFSDIR/%(INITIALGEOM)s g0.xml --noLayers --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
623  ./wrapperExtraPlots.sh -n $ALIGNMENT_ITERATION -i $ALIGNMENT_AFSDIR -0 g0.xml -z -w %(station123params)s %(dir_no_)s
624  mkdir out/extra
625  cd %(dir_no_)s
626  mv MB ../out/extra/
627  mv ME ../out/extra/
628  cd -
629 fi
630 
631 # run simple diagnostic
632 ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out --dt --csc --diagnostic
633 
634 # fill the tree browser structure:
635 ./createTree.py -i $ALIGNMENT_CAFDIR/out
636 
637 timestamp=`date \"+%%y-%%m-%%d %%H:%%M:%%S\"`
638 echo \"%(validationLabel)s.plots (${timestamp})\" > out/label.txt
639 
640 ls -l out/
641 timestamp=`date +%%Y%%m%%d%%H%%M%%S`
642 tar czf %(validationLabel)s_${timestamp}.tgz out
643 cp -f %(validationLabel)s_${timestamp}.tgz $ALIGNMENT_AFSDIR/
644 
645 """ % my_vars)
646 
647 
648 #####################################################################
649 
650 #SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS = True
651 SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS = False
652 
653 bsubfile = ["#!/bin/sh", ""]
654 bsubnames = []
655 last_align = None
656 directory = ""
657 
658 for iteration in range(1, ITERATIONS+1):
659  if iteration == 1:
660  inputdb = INITIALGEOM
661  inputdbdir = directory[:]
662  else:
663  inputdb = director + ".db"
664  inputdbdir = directory[:]
665 
666  directory = "%s%02d/" % (DIRNAME, iteration)
667  director = directory[:-1]
668 
669  dir_no_ = DIRNAME
670  if DIRNAME[-1]=='_': dir_no_ = DIRNAME[:-1]
671 
672  os.system("rm -rf %s; mkdir %s" % (directory, directory))
673  os.system("cp Alignment/MuonAlignmentAlgorithms/python/gather_cfg.py %s" % directory)
674  os.system("cp Alignment/MuonAlignmentAlgorithms/python/align_cfg.py %s" % directory)
675 
676  bsubfile.append("cd %s" % directory)
677 
678  mapplots = False
679  if mapplots_ingeneral and (iteration == 1 or iteration == 3 or iteration == 5 or iteration == 7 or iteration == 9 or iteration == ITERATIONS): mapplots = True
680  segdiffplots = False
681  if segdiffplots_ingeneral and (iteration == 1 or iteration == ITERATIONS): segdiffplots = True
682  curvatureplots = False
683  if curvatureplots_ingeneral and (iteration == 1 or iteration == ITERATIONS): curvatureplots = True
684 
685  ### gather.sh runners for njobs
686  for jobnumber in range(njobs):
687  if not options.inputInBlocks:
688  inputfiles = " ".join(fileNames[jobnumber*stepsize:(jobnumber+1)*stepsize])
689  else:
690  inputfiles = " ".join(fileNamesBlocks[jobnumber])
691 
692  if mapplots or segdiffplots or curvatureplots: copyplots = "plotting*.root"
693  else: copyplots = ""
694 
695  if len(inputfiles) > 0:
696  gather_fileName = "%sgather%03d.sh" % (directory, jobnumber)
697  writeGatherCfg(gather_fileName, vars())
698  os.system("chmod +x %s" % gather_fileName)
699  bsubfile.append("echo %sgather%03d.sh" % (directory, jobnumber))
700 
701  if last_align is None: waiter = ""
702  else: waiter = "-w \"ended(%s)\"" % last_align
703  if options.big: queue = "cmscaf1nd"
704  else: queue = "cmscaf1nh"
705 
706  bsubfile.append("bsub -R \"type==SLC6_64\" -q %s -J \"%s_gather%03d\" -u youremail.tamu.edu %s gather%03d.sh" % (queue, director, jobnumber, waiter, jobnumber))
707 
708  bsubnames.append("ended(%s_gather%03d)" % (director, jobnumber))
709 
710 
711  ### align.sh
712  if SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS:
713  if ( iteration == 1 or iteration == 3 or iteration == 5 or iteration == 7 or iteration == 9):
714  tmp = station123params, station123params, useResiduals
715  station123params, station123params, useResiduals = "000010", "000010", "0010"
716  writeAlignCfg("%salign.sh" % directory, vars())
717  station123params, station123params, useResiduals = tmp
718  elif ( iteration == 2 or iteration == 4 or iteration == 6 or iteration == 8 or iteration == 10):
719  tmp = station123params, station123params, useResiduals
720  station123params, station123params, useResiduals = "110001", "100001", "1100"
721  writeAlignCfg("%salign.sh" % directory, vars())
722  station123params, station123params, useResiduals = tmp
723  else:
724  writeAlignCfg("%salign.sh" % directory, vars())
725 
726  os.system("chmod +x %salign.sh" % directory)
727 
728  bsubfile.append("echo %salign.sh" % directory)
729  if user_mail: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_align\" -u %s -w \"%s\" align.sh" % (director, user_mail, " && ".join(bsubnames)))
730  else: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_align\" -w \"%s\" align.sh" % (director, " && ".join(bsubnames)))
731 
732  #bsubfile.append("cd ..")
733  bsubnames = []
734  last_align = "%s_align" % director
735 
736 
737  ### after the last iteration (optionally) do diagnostics run
738  if len(validationLabel) and iteration == ITERATIONS:
739  # do we have plotting files created?
740  directory1 = "%s01/" % DIRNAME
741  director1 = directory1[:-1]
742 
743  writeValidationCfg("%svalidation.sh" % directory, vars())
744  os.system("chmod +x %svalidation.sh" % directory)
745 
746  bsubfile.append("echo %svalidation.sh" % directory)
747  if user_mail: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_validation\" -u %s -w \"ended(%s)\" validation.sh" % (director, user_mail, last_align))
748  else: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_validation\" -w \"ended(%s)\" validation.sh" % (director, last_align))
749 
750  bsubfile.append("cd ..")
751  bsubfile.append("")
752 
753 
754 file(options.submitJobs, "w").write("\n".join(bsubfile))
755 os.system("chmod +x %s" % options.submitJobs)
createJobs.writeAlignCfg
def writeAlignCfg
Definition: createJobs.py:439
createJobs.writeValidationCfg
def writeValidationCfg
Definition: createJobs.py:541
spr::find
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:7
createJobs.writeGatherCfg
def writeGatherCfg
Definition: createJobs.py:370
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
Generated for CMSSW Reference Manual by doxygen 1.8.5