CMS 3D CMS Logo
createJobs.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 
3 from __future__ import print_function
4 import os, sys, optparse, math
5 
6 copyargs = sys.argv[:]
7 for i in range(len(copyargs)):
8  if copyargs[i] == "":
9  copyargs[i] = "\"\""
10  if copyargs[i].find(" ") != -1:
11  copyargs[i] = "\"%s\"" % copyargs[i]
12 commandline = " ".join(copyargs)
13 
14 prog = sys.argv[0]
15 
16 usage = """./%(prog)s DIRNAME ITERATIONS INITIALGEOM INPUTFILES [options]
17 
18 Creates (overwrites) a directory for each of the iterations and creates (overwrites)
19 submitJobs.sh with the submission sequence and dependencies.
20 
21 DIRNAME directories will be named DIRNAME01, DIRNAME02, etc.
22 ITERATIONS number of iterations
23 INITIALGEOM SQLite file containing muon geometry with tag names
24  DTAlignmentRcd, DTAlignmentErrorExtendedRcd, CSCAlignmentRcd, CSCAlignmentErrorExtendedRcd
25 INPUTFILES Python file defining 'fileNames', a list of input files as
26  strings (create with findQualityFiles.py)""" % vars()
27 
28 parser = optparse.OptionParser(usage)
29 parser.add_option("-j", "--jobs",
30  help="approximate number of \"gather\" subjobs",
31  type="int",
32  default=50,
33  dest="subjobs")
34 parser.add_option("-s", "--submitJobs",
35  help="alternate name of submitJobs.sh script (please include .sh extension); a file with this name will be OVERWRITTEN",
36  type="string",
37  default="submitJobs.sh",
38  dest="submitJobs")
39 parser.add_option("-b", "--big",
40  help="if invoked, subjobs will also be run on cmscaf1nd",
41  action="store_true",
42  dest="big")
43 parser.add_option("-u", "--user_mail",
44  help="if invoked, send mail to a specified email destination. If \"-u\" is not present, the default destination LSB_MAILTO in lsf.conf will be used",
45  type="string",
46  dest="user_mail")
47 parser.add_option("--mapplots",
48  help="if invoked, draw \"map plots\"",
49  action="store_true",
50  dest="mapplots")
51 parser.add_option("--segdiffplots",
52  help="if invoked, draw \"segment-difference plots\"",
53  action="store_true",
54  dest="segdiffplots")
55 parser.add_option("--curvatureplots",
56  help="if invoked, draw \"curvature plots\"",
57  action="store_true",
58  dest="curvatureplots")
59 parser.add_option("--globalTag",
60  help="GlobalTag for alignment/calibration conditions (typically all conditions except muon and tracker alignment)",
61  type="string",
62  default="CRAFT0831X_V1::All",
63  dest="globaltag")
64 parser.add_option("--trackerconnect",
65  help="connect string for tracker alignment (frontier://FrontierProd/CMS_COND_310X_ALIGN or sqlite_file:...)",
66  type="string",
67  default="",
68  dest="trackerconnect")
69 parser.add_option("--trackeralignment",
70  help="name of TrackerAlignmentRcd tag",
71  type="string",
72  default="Alignments",
73  dest="trackeralignment")
74 parser.add_option("--trackerAPEconnect",
75  help="connect string for tracker APEs (frontier://... or sqlite_file:...)",
76  type="string",
77  default="",
78  dest="trackerAPEconnect")
79 parser.add_option("--trackerAPE",
80  help="name of TrackerAlignmentErrorExtendedRcd tag (tracker APEs)",
81  type="string",
82  default="AlignmentErrorsExtended",
83  dest="trackerAPE")
84 parser.add_option("--trackerBowsconnect",
85  help="connect string for tracker Surface Deformations (frontier://... or sqlite_file:...)",
86  type="string",
87  default="",
88  dest="trackerBowsconnect")
89 parser.add_option("--trackerBows",
90  help="name of TrackerSurfaceDeformationRcd tag",
91  type="string",
92  default="TrackerSurfaceDeformations",
93  dest="trackerBows")
94 parser.add_option("--gprcdconnect",
95  help="connect string for GlobalPositionRcd (frontier://... or sqlite_file:...)",
96  type="string",
97  default="",
98  dest="gprcdconnect")
99 parser.add_option("--gprcd",
100  help="name of GlobalPositionRcd tag",
101  type="string",
102  default="GlobalPosition",
103  dest="gprcd")
104 parser.add_option("--iscosmics",
105  help="if invoked, use cosmic track refitter instead of the standard one",
106  action="store_true",
107  dest="iscosmics")
108 parser.add_option("--station123params",
109  help="alignable parameters for DT stations 1, 2, 3 (see SWGuideAlignmentAlgorithms#Selection_of_what_to_align)",
110  type="string",
111  default="111111",
112  dest="station123params")
113 parser.add_option("--station4params",
114  help="alignable parameters for DT station 4",
115  type="string",
116  default="100011",
117  dest="station4params")
118 parser.add_option("--cscparams",
119  help="alignable parameters for CSC chambers",
120  type="string",
121  default="100011",
122  dest="cscparams")
123 parser.add_option("--minTrackPt",
124  help="minimum allowed track transverse momentum (in GeV)",
125  type="string",
126  default="0",
127  dest="minTrackPt")
128 parser.add_option("--maxTrackPt",
129  help="maximum allowed track transverse momentum (in GeV)",
130  type="string",
131  default="1000",
132  dest="maxTrackPt")
133 parser.add_option("--minTrackP",
134  help="minimum allowed track momentum (in GeV)",
135  type="string",
136  default="0",
137  dest="minTrackP")
138 parser.add_option("--maxTrackP",
139  help="maximum allowed track momentum (in GeV)",
140  type="string",
141  default="10000",
142  dest="maxTrackP")
143 parser.add_option("--minTrackerHits",
144  help="minimum number of tracker hits",
145  type="int",
146  default=15,
147  dest="minTrackerHits")
148 parser.add_option("--maxTrackerRedChi2",
149  help="maximum tracker chi^2 per degrees of freedom",
150  type="string",
151  default="10",
152  dest="maxTrackerRedChi2")
153 parser.add_option("--notAllowTIDTEC",
154  help="if invoked, do not allow tracks that pass through the tracker's TID||TEC region (not recommended)",
155  action="store_true",
156  dest="notAllowTIDTEC")
157 parser.add_option("--twoBin",
158  help="if invoked, apply the \"two-bin method\" to control charge-antisymmetric errors",
159  action="store_true",
160  dest="twoBin")
161 parser.add_option("--weightAlignment",
162  help="if invoked, segments will be weighted by ndf/chi^2 in the alignment",
163  action="store_true",
164  dest="weightAlignment")
165 parser.add_option("--minAlignmentSegments",
166  help="minimum number of segments required to align a chamber",
167  type="int",
168  default=5,
169  dest="minAlignmentHits")
170 parser.add_option("--notCombineME11",
171  help="if invoced, treat ME1/1a and ME1/1b as separate objects",
172  action="store_true",
173  dest="notCombineME11")
174 parser.add_option("--maxEvents",
175  help="maximum number of events",
176  type="string",
177  default="-1",
178  dest="maxEvents")
179 parser.add_option("--skipEvents",
180  help="number of events to be skipped",
181  type="string",
182  default="0",
183  dest="skipEvents")
184 parser.add_option("--validationLabel",
185  help="if given nonempty string RUNLABEL, diagnostics and creation of plots will be run in the end of the last iteration; the RUNLABEL will be used to mark a run; the results will be put into a RUNLABEL_DATESTAMP.tgz tarball",
186  type="string",
187  default="",
188  dest="validationLabel")
189 parser.add_option("--maxResSlopeY",
190  help="maximum residual slope y component",
191  type="string",
192  default="10",
193  dest="maxResSlopeY")
194 parser.add_option("--motionPolicyNSigma",
195  help="minimum nsigma(deltax) position displacement in order to move a chamber for the final alignment result; default NSIGMA=3",
196  type="int",
197  default=3,
198  dest="motionPolicyNSigma")
199 parser.add_option("--noCleanUp",
200  help="if invoked, temporary plotting???.root and *.tmp files would not be removed at the end of each align job",
201  action="store_true",
202  dest="noCleanUp")
203 parser.add_option("--noCSC",
204  help="if invoked, CSC endcap chambers would not be processed",
205  action="store_true",
206  dest="noCSC")
207 parser.add_option("--noDT",
208  help="if invoked, DT barrel chambers would not be processed",
209  action="store_true",
210  dest="noDT")
211 parser.add_option("--createMapNtuple",
212  help="if invoked while mapplots are switched on, a special ntuple would be created",
213  action="store_true",
214  dest="createMapNtuple")
215 parser.add_option("--inputInBlocks",
216  help="if invoked, assume that INPUTFILES provides a list of files already groupped into job blocks, -j has no effect in that case",
217  action="store_true",
218  dest="inputInBlocks")
219 parser.add_option("--json",
220  help="If present with JSON file as argument, use JSON file for good lumi mask. "+\
221  "The latest JSON file is available at /afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions11/7TeV/Prompt/",
222  type="string",
223  default="",
224  dest="json")
225 parser.add_option("--createAlignNtuple",
226  help="if invoked, debug ntuples with residuals would be created during gather jobs",
227  action="store_true",
228  dest="createAlignNtuple")
229 parser.add_option("--residualsModel",
230  help="functional residuals model. Possible vaslues: pureGaussian2D (default), pureGaussian, GaussPowerTails, ROOTVoigt, powerLawTails",
231  type="string",
232  default="pureGaussian2D",
233  dest="residualsModel")
234 parser.add_option("--useResiduals",
235  help="select residuals to use, possible values: 1111, 1110, 1100, 1010, 0010 that correspond to x y dxdz dydz residuals",
236  type="string",
237  default="1110",
238  dest="useResiduals")
239 parser.add_option("--peakNSigma",
240  help="if >0, only residuals peaks within n-sigma multidimentional ellipsoid would be considered in the alignment fit",
241  type="string",
242  default="-1.",
243  dest="peakNSigma")
244 parser.add_option("--preFilter",
245  help="if invoked, MuonAlignmentPreFilter module would be invoked in the Path's beginning. Can significantly speed up gather jobs.",
246  action="store_true",
247  dest="preFilter")
248 parser.add_option("--muonCollectionTag",
249  help="If empty, use trajectories. If not empty, it's InputTag of muons collection to use in tracker muons based approach, e.g., 'newmuons' or 'muons'",
250  type="string",
251  default="",
252  dest="muonCollectionTag")
253 parser.add_option("--maxDxy",
254  help="maximum track impact parameter with relation to beamline",
255  type="string",
256  default="1000.",
257  dest="maxDxy")
258 parser.add_option("--minNCrossedChambers",
259  help="minimum number of muon chambers that a track is required to cross",
260  type="string",
261  default="3",
262  dest="minNCrossedChambers")
263 parser.add_option("--extraPlots",
264  help="produce additional plots with geometry, reports differences, and corrections visulizations",
265  action="store_true",
266  dest="extraPlots")
267 
268 if len(sys.argv) < 5:
269  raise SystemError("Too few arguments.\n\n"+parser.format_help())
270 
271 DIRNAME = sys.argv[1]
272 ITERATIONS = int(sys.argv[2])
273 INITIALGEOM = sys.argv[3]
274 INPUTFILES = sys.argv[4]
275 
276 options, args = parser.parse_args(sys.argv[5:])
277 user_mail = options.user_mail
278 mapplots_ingeneral = options.mapplots
279 segdiffplots_ingeneral = options.segdiffplots
280 curvatureplots_ingeneral = options.curvatureplots
281 globaltag = options.globaltag
282 trackerconnect = options.trackerconnect
283 trackeralignment = options.trackeralignment
284 trackerAPEconnect = options.trackerAPEconnect
285 trackerAPE = options.trackerAPE
286 trackerBowsconnect = options.trackerBowsconnect
287 trackerBows = options.trackerBows
288 gprcdconnect = options.gprcdconnect
289 gprcd = options.gprcd
290 iscosmics = str(options.iscosmics)
291 station123params = options.station123params
292 station4params = options.station4params
293 cscparams = options.cscparams
294 muonCollectionTag = options.muonCollectionTag
295 minTrackPt = options.minTrackPt
296 maxTrackPt = options.maxTrackPt
297 minTrackP = options.minTrackP
298 maxTrackP = options.maxTrackP
299 maxDxy = options.maxDxy
300 minTrackerHits = str(options.minTrackerHits)
301 maxTrackerRedChi2 = options.maxTrackerRedChi2
302 minNCrossedChambers = options.minNCrossedChambers
303 allowTIDTEC = str(not options.notAllowTIDTEC)
304 twoBin = str(options.twoBin)
305 weightAlignment = str(options.weightAlignment)
306 minAlignmentHits = str(options.minAlignmentHits)
307 combineME11 = str(not options.notCombineME11)
308 maxEvents = options.maxEvents
309 skipEvents = options.skipEvents
310 validationLabel = options.validationLabel
311 maxResSlopeY = options.maxResSlopeY
312 theNSigma = options.motionPolicyNSigma
313 residualsModel = options.residualsModel
314 peakNSigma = options.peakNSigma
315 preFilter = not not options.preFilter
316 extraPlots = options.extraPlots
317 useResiduals = options.useResiduals
318 
319 
320 #print "check: ", allowTIDTEC, combineME11, preFilter
321 
322 doCleanUp = not options.noCleanUp
323 createMapNtuple = not not options.createMapNtuple
324 createAlignNtuple = not not options.createAlignNtuple
325 
326 doCSC = True
327 if options.noCSC: doCSC = False
328 doDT = True
329 if options.noDT: doDT = False
330 if options.noCSC and options.noDT:
331  print("cannot do --noCSC and --noDT at the same time!")
332  sys.exit()
333 
334 json_file = options.json
335 
336 fileNames=[]
337 fileNamesBlocks=[]
338 execfile(INPUTFILES)
339 njobs = options.subjobs
340 if (options.inputInBlocks):
341  njobs = len(fileNamesBlocks)
342  if njobs==0:
343  print("while --inputInBlocks is specified, the INPUTFILES has no blocks!")
344  sys.exit()
345 
346 stepsize = int(math.ceil(1.*len(fileNames)/options.subjobs))
347 
348 pwd = str(os.getcwd())
349 
350 copytrackerdb = ""
351 if trackerconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerconnect[12:]
352 if trackerAPEconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerAPEconnect[12:]
353 if trackerBowsconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerBowsconnect[12:]
354 if gprcdconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % gprcdconnect[12:]
355 
356 
357 #####################################################################
358 # step 0: convert initial geometry to xml
359 INITIALXML = INITIALGEOM + '.xml'
360 if INITIALGEOM[-3:]=='.db':
361  INITIALXML = INITIALGEOM[:-3] + '.xml'
362 print("Converting",INITIALGEOM,"to",INITIALXML," ...will be done in several seconds...")
363 print("./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %s %s --gprcdconnect %s --gprcd %s" % (INITIALGEOM,INITIALXML,gprcdconnect,gprcd))
364 exit_code = os.system("./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %s %s --gprcdconnect %s --gprcd %s" % (INITIALGEOM,INITIALXML,gprcdconnect,gprcd))
365 if exit_code>0:
366  print("problem: conversion exited with code:", exit_code)
367  sys.exit()
368 
369 #####################################################################
370 
371 def writeGatherCfg(fname, my_vars):
372  file(fname, "w").write("""#/bin/sh
373 # %(commandline)s
374 
375 export ALIGNMENT_CAFDIR=`pwd`
376 
377 cd %(pwd)s
378 eval `scramv1 run -sh`
379 export ALIGNMENT_AFSDIR=`pwd`
380 
381 export ALIGNMENT_INPUTFILES='%(inputfiles)s'
382 export ALIGNMENT_ITERATION=%(iteration)d
383 export ALIGNMENT_JOBNUMBER=%(jobnumber)d
384 export ALIGNMENT_MAPPLOTS=%(mapplots)s
385 export ALIGNMENT_SEGDIFFPLOTS=%(segdiffplots)s
386 export ALIGNMENT_CURVATUREPLOTS=%(curvatureplots)s
387 export ALIGNMENT_GLOBALTAG=%(globaltag)s
388 export ALIGNMENT_INPUTDB=%(inputdb)s
389 export ALIGNMENT_TRACKERCONNECT=%(trackerconnect)s
390 export ALIGNMENT_TRACKERALIGNMENT=%(trackeralignment)s
391 export ALIGNMENT_TRACKERAPECONNECT=%(trackerAPEconnect)s
392 export ALIGNMENT_TRACKERAPE=%(trackerAPE)s
393 export ALIGNMENT_TRACKERBOWSCONNECT=%(trackerBowsconnect)s
394 export ALIGNMENT_TRACKERBOWS=%(trackerBows)s
395 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
396 export ALIGNMENT_GPRCD=%(gprcd)s
397 export ALIGNMENT_ISCOSMICS=%(iscosmics)s
398 export ALIGNMENT_STATION123PARAMS=%(station123params)s
399 export ALIGNMENT_STATION4PARAMS=%(station4params)s
400 export ALIGNMENT_CSCPARAMS=%(cscparams)s
401 export ALIGNMENT_MUONCOLLECTIONTAG=%(muonCollectionTag)s
402 export ALIGNMENT_MINTRACKPT=%(minTrackPt)s
403 export ALIGNMENT_MAXTRACKPT=%(maxTrackPt)s
404 export ALIGNMENT_MINTRACKP=%(minTrackP)s
405 export ALIGNMENT_MAXTRACKP=%(maxTrackP)s
406 export ALIGNMENT_MAXDXY=%(maxDxy)s
407 export ALIGNMENT_MINTRACKERHITS=%(minTrackerHits)s
408 export ALIGNMENT_MAXTRACKERREDCHI2=%(maxTrackerRedChi2)s
409 export ALIGNMENT_MINNCROSSEDCHAMBERS=%(minNCrossedChambers)s
410 export ALIGNMENT_ALLOWTIDTEC=%(allowTIDTEC)s
411 export ALIGNMENT_TWOBIN=%(twoBin)s
412 export ALIGNMENT_WEIGHTALIGNMENT=%(weightAlignment)s
413 export ALIGNMENT_MINALIGNMENTHITS=%(minAlignmentHits)s
414 export ALIGNMENT_COMBINEME11=%(combineME11)s
415 export ALIGNMENT_MAXEVENTS=%(maxEvents)s
416 export ALIGNMENT_SKIPEVENTS=%(skipEvents)s
417 export ALIGNMENT_MAXRESSLOPEY=%(maxResSlopeY)s
418 export ALIGNMENT_DO_DT=%(doDT)s
419 export ALIGNMENT_DO_CSC=%(doCSC)s
420 export ALIGNMENT_JSON=%(json_file)s
421 export ALIGNMENT_CREATEMAPNTUPLE=%(createMapNtuple)s
422 #export ALIGNMENT_CREATEALIGNNTUPLE=%(createAlignNtuple)s
423 export ALIGNMENT_PREFILTER=%(preFilter)s
424 
425 
426 if [ \"zzz$ALIGNMENT_JSON\" != \"zzz\" ]; then
427  cp -f $ALIGNMENT_JSON $ALIGNMENT_CAFDIR/
428 fi
429 
430 cp -f %(directory)sgather_cfg.py %(inputdbdir)s%(inputdb)s %(copytrackerdb)s $ALIGNMENT_CAFDIR/
431 cd $ALIGNMENT_CAFDIR/
432 ls -l
433 cmsRun gather_cfg.py
434 ls -l
435 cp -f *.tmp %(copyplots)s $ALIGNMENT_AFSDIR/%(directory)s
436 """ % my_vars)
437 
438 #####################################################################
439 
440 def writeAlignCfg(fname, my_vars):
441  file("%salign.sh" % directory, "w").write("""#!/bin/sh
442 # %(commandline)s
443 
444 export ALIGNMENT_CAFDIR=`pwd`
445 
446 cd %(pwd)s
447 eval `scramv1 run -sh`
448 export ALIGNMENT_AFSDIR=`pwd`
449 export ALIGNMENT_INPUTDB=%(inputdb)s
450 export ALIGNMENT_ITERATION=%(iteration)d
451 export ALIGNMENT_GLOBALTAG=%(globaltag)s
452 export ALIGNMENT_TRACKERCONNECT=%(trackerconnect)s
453 export ALIGNMENT_TRACKERALIGNMENT=%(trackeralignment)s
454 export ALIGNMENT_TRACKERAPECONNECT=%(trackerAPEconnect)s
455 export ALIGNMENT_TRACKERAPE=%(trackerAPE)s
456 export ALIGNMENT_TRACKERBOWSCONNECT=%(trackerBowsconnect)s
457 export ALIGNMENT_TRACKERBOWS=%(trackerBows)s
458 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
459 export ALIGNMENT_GPRCD=%(gprcd)s
460 export ALIGNMENT_ISCOSMICS=%(iscosmics)s
461 export ALIGNMENT_STATION123PARAMS=%(station123params)s
462 export ALIGNMENT_STATION4PARAMS=%(station4params)s
463 export ALIGNMENT_CSCPARAMS=%(cscparams)s
464 export ALIGNMENT_MINTRACKPT=%(minTrackPt)s
465 export ALIGNMENT_MAXTRACKPT=%(maxTrackPt)s
466 export ALIGNMENT_MINTRACKP=%(minTrackP)s
467 export ALIGNMENT_MAXTRACKP=%(maxTrackP)s
468 export ALIGNMENT_MINTRACKERHITS=%(minTrackerHits)s
469 export ALIGNMENT_MAXTRACKERREDCHI2=%(maxTrackerRedChi2)s
470 export ALIGNMENT_ALLOWTIDTEC=%(allowTIDTEC)s
471 export ALIGNMENT_TWOBIN=%(twoBin)s
472 export ALIGNMENT_WEIGHTALIGNMENT=%(weightAlignment)s
473 export ALIGNMENT_MINALIGNMENTHITS=%(minAlignmentHits)s
474 export ALIGNMENT_COMBINEME11=%(combineME11)s
475 export ALIGNMENT_MAXRESSLOPEY=%(maxResSlopeY)s
476 export ALIGNMENT_CLEANUP=%(doCleanUp)s
477 export ALIGNMENT_CREATEALIGNNTUPLE=%(createAlignNtuple)s
478 export ALIGNMENT_RESIDUALSMODEL=%(residualsModel)s
479 export ALIGNMENT_PEAKNSIGMA=%(peakNSigma)s
480 export ALIGNMENT_USERESIDUALS=%(useResiduals)s
481 
482 cp -f %(directory)salign_cfg.py %(inputdbdir)s%(inputdb)s %(directory)s*.tmp %(copytrackerdb)s $ALIGNMENT_CAFDIR/
483 
484 export ALIGNMENT_PLOTTINGTMP=`find %(directory)splotting0*.root -maxdepth 1 -size +0 -print 2> /dev/null`
485 
486 # if it's 1st or last iteration, combine _plotting.root files into one:
487 if [ \"$ALIGNMENT_ITERATION\" != \"111\" ] || [ \"$ALIGNMENT_ITERATION\" == \"%(ITERATIONS)s\" ]; then
488  #nfiles=$(ls %(directory)splotting0*.root 2> /dev/null | wc -l)
489  if [ \"zzz$ALIGNMENT_PLOTTINGTMP\" != \"zzz\" ]; then
490  hadd -f1 %(directory)s%(director)s_plotting.root %(directory)splotting0*.root
491  #if [ $? == 0 ] && [ \"$ALIGNMENT_CLEANUP\" == \"True\" ]; then rm %(directory)splotting0*.root; fi
492  fi
493 fi
494 
495 if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ \"zzz$ALIGNMENT_PLOTTINGTMP\" != \"zzz\" ]; then
496  rm $ALIGNMENT_PLOTTINGTMP
497 fi
498 
499 cd $ALIGNMENT_CAFDIR/
500 export ALIGNMENT_ALIGNMENTTMP=`find alignment*.tmp -maxdepth 1 -size +1k -print 2> /dev/null`
501 ls -l
502 
503 cmsRun align_cfg.py
504 cp -f MuonAlignmentFromReference_report.py $ALIGNMENT_AFSDIR/%(directory)s%(director)s_report.py
505 cp -f MuonAlignmentFromReference_outputdb.db $ALIGNMENT_AFSDIR/%(directory)s%(director)s.db
506 cp -f MuonAlignmentFromReference_plotting.root $ALIGNMENT_AFSDIR/%(directory)s%(director)s.root
507 
508 cd $ALIGNMENT_AFSDIR
509 ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s.db %(directory)s%(director)s.xml --noLayers --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
510 
511 export ALIGNMENT_ALIGNMENTTMP=`find %(directory)salignment*.tmp -maxdepth 1 -size +1k -print 2> /dev/null`
512 if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ \"zzz$ALIGNMENT_ALIGNMENTTMP\" != \"zzz\" ]; then
513  rm $ALIGNMENT_ALIGNMENTTMP
514  echo " "
515 fi
516 
517 # if it's not 1st or last iteration, do some clean up:
518 if [ \"$ALIGNMENT_ITERATION\" != \"1\" ] && [ \"$ALIGNMENT_ITERATION\" != \"%(ITERATIONS)s\" ]; then
519  if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ -e %(directory)s%(director)s.root ]; then
520  rm %(directory)s%(director)s.root
521  fi
522 fi
523 
524 # if it's last iteration, apply chamber motion policy
525 if [ \"$ALIGNMENT_ITERATION\" == \"%(ITERATIONS)s\" ]; then
526  # convert this iteration's geometry into detailed xml
527  ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s.db %(directory)s%(director)s_extra.xml --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
528  # perform motion policy
529  ./Alignment/MuonAlignmentAlgorithms/scripts/motionPolicyChamber.py \
530  %(INITIALXML)s %(directory)s%(director)s_extra.xml \
531  %(directory)s%(director)s_report.py \
532  %(directory)s%(director)s_final.xml \
533  --nsigma %(theNSigma)s
534  # convert the resulting xml into the final sqlite geometry
535  ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s_final.xml %(directory)s%(director)s_final.db --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
536 fi
537 
538 """ % my_vars)
539 
540 #####################################################################
541 
542 def writeValidationCfg(fname, my_vars):
543  file(fname, "w").write("""#!/bin/sh
544 # %(commandline)s
545 
546 export ALIGNMENT_CAFDIR=`pwd`
547 mkdir files
548 mkdir out
549 
550 cd %(pwd)s
551 eval `scramv1 run -sh`
552 ALIGNMENT_AFSDIR=`pwd`
553 ALIGNMENT_ITERATION=%(iteration)d
554 ALIGNMENT_MAPPLOTS=None
555 ALIGNMENT_SEGDIFFPLOTS=None
556 ALIGNMENT_CURVATUREPLOTS=None
557 ALIGNMENT_EXTRAPLOTS=%(extraPlots)s
558 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
559 export ALIGNMENT_GPRCD=%(gprcd)s
560 export ALIGNMENT_DO_DT=%(doDT)s
561 export ALIGNMENT_DO_CSC=%(doCSC)s
562 
563 
564 # copy the scripts to CAFDIR
565 cd Alignment/MuonAlignmentAlgorithms/scripts/
566 cp -f plotscripts.py $ALIGNMENT_CAFDIR/
567 cp -f mutypes.py $ALIGNMENT_CAFDIR/
568 cp -f alignmentValidation.py $ALIGNMENT_CAFDIR/
569 cp -f phiedges_fitfunctions.C $ALIGNMENT_CAFDIR/
570 cp -f createTree.py $ALIGNMENT_CAFDIR/
571 cp -f signConventions.py $ALIGNMENT_CAFDIR/
572 cp -f convertSQLiteXML.py $ALIGNMENT_CAFDIR/
573 cp -f wrapperExtraPlots.sh $ALIGNMENT_CAFDIR/
574 cd -
575 cp Alignment/MuonAlignmentAlgorithms/test/browser/tree* $ALIGNMENT_CAFDIR/out/
576 
577 # copy the results to CAFDIR
578 cp -f %(directory1)s%(director1)s_report.py $ALIGNMENT_CAFDIR/files/
579 cp -f %(directory)s%(director)s_report.py $ALIGNMENT_CAFDIR/files/
580 cp -f %(directory1)s%(director1)s.root $ALIGNMENT_CAFDIR/files/
581 cp -f %(directory)s%(director)s.root $ALIGNMENT_CAFDIR/files/
582 if [ -e %(directory1)s%(director1)s_plotting.root ] && [ -e %(directory)s%(director)s_plotting.root ]; then
583  cp -f %(directory1)s%(director1)s_plotting.root $ALIGNMENT_CAFDIR/files/
584  cp -f %(directory)s%(director)s_plotting.root $ALIGNMENT_CAFDIR/files/
585  ALIGNMENT_MAPPLOTS=%(mapplots)s
586  ALIGNMENT_SEGDIFFPLOTS=%(segdiffplots)s
587  ALIGNMENT_CURVATUREPLOTS=%(curvatureplots)s
588 fi
589 
590 dtcsc=""
591 if [ $ALIGNMENT_DO_DT == \"True\" ]; then
592  dtcsc="--dt"
593 fi
594 if [ $ALIGNMENT_DO_CSC == \"True\" ]; then
595  dtcsc="${dtcsc} --csc"
596 fi
597 
598 
599 cd $ALIGNMENT_CAFDIR/
600 echo \" ### Start running ###\"
601 date
602 
603 # do fits and median plots first
604 ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out --createDirSructure --dt --csc --fit --median
605 
606 if [ $ALIGNMENT_MAPPLOTS == \"True\" ]; then
607  ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out $dtcsc --map
608 fi
609 
610 if [ $ALIGNMENT_SEGDIFFPLOTS == \"True\" ]; then
611  ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out $dtcsc --segdiff
612 fi
613 
614 if [ $ALIGNMENT_CURVATUREPLOTS == \"True\" ]; then
615  ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out $dtcsc --curvature
616 fi
617 
618 if [ $ALIGNMENT_EXTRAPLOTS == \"True\" ]; then
619  if [ \"zzz%(copytrackerdb)s\" != \"zzz\" ]; then
620  cp -f $ALIGNMENT_AFSDIR/%(copytrackerdb)s $ALIGNMENT_CAFDIR/
621  fi
622  cp $ALIGNMENT_AFSDIR/inertGlobalPositionRcd.db .
623  ./convertSQLiteXML.py $ALIGNMENT_AFSDIR/%(INITIALGEOM)s g0.xml --noLayers --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
624  ./wrapperExtraPlots.sh -n $ALIGNMENT_ITERATION -i $ALIGNMENT_AFSDIR -0 g0.xml -z -w %(station123params)s %(dir_no_)s
625  mkdir out/extra
626  cd %(dir_no_)s
627  mv MB ../out/extra/
628  mv ME ../out/extra/
629  cd -
630 fi
631 
632 # run simple diagnostic
633 ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out --dt --csc --diagnostic
634 
635 # fill the tree browser structure:
636 ./createTree.py -i $ALIGNMENT_CAFDIR/out
637 
638 timestamp=`date \"+%%y-%%m-%%d %%H:%%M:%%S\"`
639 echo \"%(validationLabel)s.plots (${timestamp})\" > out/label.txt
640 
641 ls -l out/
642 timestamp=`date +%%Y%%m%%d%%H%%M%%S`
643 tar czf %(validationLabel)s_${timestamp}.tgz out
644 cp -f %(validationLabel)s_${timestamp}.tgz $ALIGNMENT_AFSDIR/
645 
646 """ % my_vars)
647 
648 
649 #####################################################################
650 
651 #SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS = True
652 SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS = False
653 
654 bsubfile = ["#!/bin/sh", ""]
655 bsubnames = []
656 last_align = None
657 directory = ""
658 
659 for iteration in range(1, ITERATIONS+1):
660  if iteration == 1:
661  inputdb = INITIALGEOM
662  inputdbdir = directory[:]
663  else:
664  inputdb = director + ".db"
665  inputdbdir = directory[:]
666 
667  directory = "%s%02d/" % (DIRNAME, iteration)
668  director = directory[:-1]
669 
670  dir_no_ = DIRNAME
671  if DIRNAME[-1]=='_': dir_no_ = DIRNAME[:-1]
672 
673  os.system("rm -rf %s; mkdir %s" % (directory, directory))
674  os.system("cp Alignment/MuonAlignmentAlgorithms/python/gather_cfg.py %s" % directory)
675  os.system("cp Alignment/MuonAlignmentAlgorithms/python/align_cfg.py %s" % directory)
676 
677  bsubfile.append("cd %s" % directory)
678 
679  mapplots = False
680  if mapplots_ingeneral and (iteration == 1 or iteration == 3 or iteration == 5 or iteration == 7 or iteration == 9 or iteration == ITERATIONS): mapplots = True
681  segdiffplots = False
682  if segdiffplots_ingeneral and (iteration == 1 or iteration == ITERATIONS): segdiffplots = True
683  curvatureplots = False
684  if curvatureplots_ingeneral and (iteration == 1 or iteration == ITERATIONS): curvatureplots = True
685 
686  ### gather.sh runners for njobs
687  for jobnumber in range(njobs):
688  if not options.inputInBlocks:
689  inputfiles = " ".join(fileNames[jobnumber*stepsize:(jobnumber+1)*stepsize])
690  else:
691  inputfiles = " ".join(fileNamesBlocks[jobnumber])
692 
693  if mapplots or segdiffplots or curvatureplots: copyplots = "plotting*.root"
694  else: copyplots = ""
695 
696  if len(inputfiles) > 0:
697  gather_fileName = "%sgather%03d.sh" % (directory, jobnumber)
698  writeGatherCfg(gather_fileName, vars())
699  os.system("chmod +x %s" % gather_fileName)
700  bsubfile.append("echo %sgather%03d.sh" % (directory, jobnumber))
701 
702  if last_align is None: waiter = ""
703  else: waiter = "-w \"ended(%s)\"" % last_align
704  if options.big: queue = "cmscaf1nd"
705  else: queue = "cmscaf1nh"
706 
707  bsubfile.append("bsub -R \"type==SLC6_64\" -q %s -J \"%s_gather%03d\" -u youremail.tamu.edu %s gather%03d.sh" % (queue, director, jobnumber, waiter, jobnumber))
708 
709  bsubnames.append("ended(%s_gather%03d)" % (director, jobnumber))
710 
711 
712  ### align.sh
713  if SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS:
714  if ( iteration == 1 or iteration == 3 or iteration == 5 or iteration == 7 or iteration == 9):
715  tmp = station123params, station123params, useResiduals
716  station123params, station123params, useResiduals = "000010", "000010", "0010"
717  writeAlignCfg("%salign.sh" % directory, vars())
718  station123params, station123params, useResiduals = tmp
719  elif ( iteration == 2 or iteration == 4 or iteration == 6 or iteration == 8 or iteration == 10):
720  tmp = station123params, station123params, useResiduals
721  station123params, station123params, useResiduals = "110001", "100001", "1100"
722  writeAlignCfg("%salign.sh" % directory, vars())
723  station123params, station123params, useResiduals = tmp
724  else:
725  writeAlignCfg("%salign.sh" % directory, vars())
726 
727  os.system("chmod +x %salign.sh" % directory)
728 
729  bsubfile.append("echo %salign.sh" % directory)
730  if user_mail: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_align\" -u %s -w \"%s\" align.sh" % (director, user_mail, " && ".join(bsubnames)))
731  else: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_align\" -w \"%s\" align.sh" % (director, " && ".join(bsubnames)))
732 
733  #bsubfile.append("cd ..")
734  bsubnames = []
735  last_align = "%s_align" % director
736 
737 
738  ### after the last iteration (optionally) do diagnostics run
739  if len(validationLabel) and iteration == ITERATIONS:
740  # do we have plotting files created?
741  directory1 = "%s01/" % DIRNAME
742  director1 = directory1[:-1]
743 
744  writeValidationCfg("%svalidation.sh" % directory, vars())
745  os.system("chmod +x %svalidation.sh" % directory)
746 
747  bsubfile.append("echo %svalidation.sh" % directory)
748  if user_mail: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_validation\" -u %s -w \"ended(%s)\" validation.sh" % (director, user_mail, last_align))
749  else: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_validation\" -w \"ended(%s)\" validation.sh" % (director, last_align))
750 
751  bsubfile.append("cd ..")
752  bsubfile.append("")
753 
754 
755 file(options.submitJobs, "w").write("\n".join(bsubfile))
756 os.system("chmod +x %s" % options.submitJobs)
createJobs.writeAlignCfg
def writeAlignCfg(fname, my_vars)
Definition: createJobs.py:440
edm::print
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
createJobs.writeGatherCfg
def writeGatherCfg(fname, my_vars)
Definition: createJobs.py:371
spr::find
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:20
createJobs.writeValidationCfg
def writeValidationCfg(fname, my_vars)
Definition: createJobs.py:542
FrontierConditions_GlobalTag_cff.file
file
Definition: FrontierConditions_GlobalTag_cff.py:13
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
TriggerAnalyzer.write
def write(self, setup)
Definition: TriggerAnalyzer.py:98
str
#define str(s)
Definition: TestProcessor.cc:47
Generated for CMSSW Reference Manual by doxygen 1.8.11