CMS 3D CMS Logo
createJobs.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 
3 from __future__ import print_function
4 from builtins import range
5 import os, sys, optparse, math
6 
7 copyargs = sys.argv[:]
8 for i in range(len(copyargs)):
9  if copyargs[i] == "":
10  copyargs[i] = "\"\""
11  if copyargs[i].find(" ") != -1:
12  copyargs[i] = "\"%s\"" % copyargs[i]
13 commandline = " ".join(copyargs)
14 
15 prog = sys.argv[0]
16 
17 usage = """./%(prog)s DIRNAME ITERATIONS INITIALGEOM INPUTFILES [options]
18 
19 Creates (overwrites) a directory for each of the iterations and creates (overwrites)
20 submitJobs.sh with the submission sequence and dependencies.
21 
22 DIRNAME directories will be named DIRNAME01, DIRNAME02, etc.
23 ITERATIONS number of iterations
24 INITIALGEOM SQLite file containing muon geometry with tag names
25  DTAlignmentRcd, DTAlignmentErrorExtendedRcd, CSCAlignmentRcd, CSCAlignmentErrorExtendedRcd
26 INPUTFILES Python file defining 'fileNames', a list of input files as
27  strings (create with findQualityFiles.py)""" % vars()
28 
29 parser = optparse.OptionParser(usage)
30 parser.add_option("-j", "--jobs",
31  help="approximate number of \"gather\" subjobs",
32  type="int",
33  default=50,
34  dest="subjobs")
35 parser.add_option("-s", "--submitJobs",
36  help="alternate name of submitJobs.sh script (please include .sh extension); a file with this name will be OVERWRITTEN",
37  type="string",
38  default="submitJobs.sh",
39  dest="submitJobs")
40 parser.add_option("-b", "--big",
41  help="if invoked, subjobs will also be run on cmscaf1nd",
42  action="store_true",
43  dest="big")
44 parser.add_option("-u", "--user_mail",
45  help="if invoked, send mail to a specified email destination. If \"-u\" is not present, the default destination LSB_MAILTO in lsf.conf will be used",
46  type="string",
47  dest="user_mail")
48 parser.add_option("--mapplots",
49  help="if invoked, draw \"map plots\"",
50  action="store_true",
51  dest="mapplots")
52 parser.add_option("--segdiffplots",
53  help="if invoked, draw \"segment-difference plots\"",
54  action="store_true",
55  dest="segdiffplots")
56 parser.add_option("--curvatureplots",
57  help="if invoked, draw \"curvature plots\"",
58  action="store_true",
59  dest="curvatureplots")
60 parser.add_option("--globalTag",
61  help="GlobalTag for alignment/calibration conditions (typically all conditions except muon and tracker alignment)",
62  type="string",
63  default="CRAFT0831X_V1::All",
64  dest="globaltag")
65 parser.add_option("--trackerconnect",
66  help="connect string for tracker alignment (frontier://FrontierProd/CMS_COND_310X_ALIGN or sqlite_file:...)",
67  type="string",
68  default="",
69  dest="trackerconnect")
70 parser.add_option("--trackeralignment",
71  help="name of TrackerAlignmentRcd tag",
72  type="string",
73  default="Alignments",
74  dest="trackeralignment")
75 parser.add_option("--trackerAPEconnect",
76  help="connect string for tracker APEs (frontier://... or sqlite_file:...)",
77  type="string",
78  default="",
79  dest="trackerAPEconnect")
80 parser.add_option("--trackerAPE",
81  help="name of TrackerAlignmentErrorExtendedRcd tag (tracker APEs)",
82  type="string",
83  default="AlignmentErrorsExtended",
84  dest="trackerAPE")
85 parser.add_option("--trackerBowsconnect",
86  help="connect string for tracker Surface Deformations (frontier://... or sqlite_file:...)",
87  type="string",
88  default="",
89  dest="trackerBowsconnect")
90 parser.add_option("--trackerBows",
91  help="name of TrackerSurfaceDeformationRcd tag",
92  type="string",
93  default="TrackerSurfaceDeformations",
94  dest="trackerBows")
95 parser.add_option("--gprcdconnect",
96  help="connect string for GlobalPositionRcd (frontier://... or sqlite_file:...)",
97  type="string",
98  default="",
99  dest="gprcdconnect")
100 parser.add_option("--gprcd",
101  help="name of GlobalPositionRcd tag",
102  type="string",
103  default="GlobalPosition",
104  dest="gprcd")
105 parser.add_option("--iscosmics",
106  help="if invoked, use cosmic track refitter instead of the standard one",
107  action="store_true",
108  dest="iscosmics")
109 parser.add_option("--station123params",
110  help="alignable parameters for DT stations 1, 2, 3 (see SWGuideAlignmentAlgorithms#Selection_of_what_to_align)",
111  type="string",
112  default="111111",
113  dest="station123params")
114 parser.add_option("--station4params",
115  help="alignable parameters for DT station 4",
116  type="string",
117  default="100011",
118  dest="station4params")
119 parser.add_option("--cscparams",
120  help="alignable parameters for CSC chambers",
121  type="string",
122  default="100011",
123  dest="cscparams")
124 parser.add_option("--minTrackPt",
125  help="minimum allowed track transverse momentum (in GeV)",
126  type="string",
127  default="0",
128  dest="minTrackPt")
129 parser.add_option("--maxTrackPt",
130  help="maximum allowed track transverse momentum (in GeV)",
131  type="string",
132  default="1000",
133  dest="maxTrackPt")
134 parser.add_option("--minTrackP",
135  help="minimum allowed track momentum (in GeV)",
136  type="string",
137  default="0",
138  dest="minTrackP")
139 parser.add_option("--maxTrackP",
140  help="maximum allowed track momentum (in GeV)",
141  type="string",
142  default="10000",
143  dest="maxTrackP")
144 parser.add_option("--minTrackerHits",
145  help="minimum number of tracker hits",
146  type="int",
147  default=15,
148  dest="minTrackerHits")
149 parser.add_option("--maxTrackerRedChi2",
150  help="maximum tracker chi^2 per degrees of freedom",
151  type="string",
152  default="10",
153  dest="maxTrackerRedChi2")
154 parser.add_option("--notAllowTIDTEC",
155  help="if invoked, do not allow tracks that pass through the tracker's TID||TEC region (not recommended)",
156  action="store_true",
157  dest="notAllowTIDTEC")
158 parser.add_option("--twoBin",
159  help="if invoked, apply the \"two-bin method\" to control charge-antisymmetric errors",
160  action="store_true",
161  dest="twoBin")
162 parser.add_option("--weightAlignment",
163  help="if invoked, segments will be weighted by ndf/chi^2 in the alignment",
164  action="store_true",
165  dest="weightAlignment")
166 parser.add_option("--minAlignmentSegments",
167  help="minimum number of segments required to align a chamber",
168  type="int",
169  default=5,
170  dest="minAlignmentHits")
171 parser.add_option("--notCombineME11",
172  help="if invoced, treat ME1/1a and ME1/1b as separate objects",
173  action="store_true",
174  dest="notCombineME11")
175 parser.add_option("--maxEvents",
176  help="maximum number of events",
177  type="string",
178  default="-1",
179  dest="maxEvents")
180 parser.add_option("--skipEvents",
181  help="number of events to be skipped",
182  type="string",
183  default="0",
184  dest="skipEvents")
185 parser.add_option("--validationLabel",
186  help="if given nonempty string RUNLABEL, diagnostics and creation of plots will be run in the end of the last iteration; the RUNLABEL will be used to mark a run; the results will be put into a RUNLABEL_DATESTAMP.tgz tarball",
187  type="string",
188  default="",
189  dest="validationLabel")
190 parser.add_option("--maxResSlopeY",
191  help="maximum residual slope y component",
192  type="string",
193  default="10",
194  dest="maxResSlopeY")
195 parser.add_option("--motionPolicyNSigma",
196  help="minimum nsigma(deltax) position displacement in order to move a chamber for the final alignment result; default NSIGMA=3",
197  type="int",
198  default=3,
199  dest="motionPolicyNSigma")
200 parser.add_option("--noCleanUp",
201  help="if invoked, temporary plotting???.root and *.tmp files would not be removed at the end of each align job",
202  action="store_true",
203  dest="noCleanUp")
204 parser.add_option("--noCSC",
205  help="if invoked, CSC endcap chambers would not be processed",
206  action="store_true",
207  dest="noCSC")
208 parser.add_option("--noDT",
209  help="if invoked, DT barrel chambers would not be processed",
210  action="store_true",
211  dest="noDT")
212 parser.add_option("--createMapNtuple",
213  help="if invoked while mapplots are switched on, a special ntuple would be created",
214  action="store_true",
215  dest="createMapNtuple")
216 parser.add_option("--inputInBlocks",
217  help="if invoked, assume that INPUTFILES provides a list of files already groupped into job blocks, -j has no effect in that case",
218  action="store_true",
219  dest="inputInBlocks")
220 parser.add_option("--json",
221  help="If present with JSON file as argument, use JSON file for good lumi mask. "+\
222  "The latest JSON file is available at /afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions11/7TeV/Prompt/",
223  type="string",
224  default="",
225  dest="json")
226 parser.add_option("--createAlignNtuple",
227  help="if invoked, debug ntuples with residuals would be created during gather jobs",
228  action="store_true",
229  dest="createAlignNtuple")
230 parser.add_option("--residualsModel",
231  help="functional residuals model. Possible vaslues: pureGaussian2D (default), pureGaussian, GaussPowerTails, ROOTVoigt, powerLawTails",
232  type="string",
233  default="pureGaussian2D",
234  dest="residualsModel")
235 parser.add_option("--useResiduals",
236  help="select residuals to use, possible values: 1111, 1110, 1100, 1010, 0010 that correspond to x y dxdz dydz residuals",
237  type="string",
238  default="1110",
239  dest="useResiduals")
240 parser.add_option("--peakNSigma",
241  help="if >0, only residuals peaks within n-sigma multidimentional ellipsoid would be considered in the alignment fit",
242  type="string",
243  default="-1.",
244  dest="peakNSigma")
245 parser.add_option("--preFilter",
246  help="if invoked, MuonAlignmentPreFilter module would be invoked in the Path's beginning. Can significantly speed up gather jobs.",
247  action="store_true",
248  dest="preFilter")
249 parser.add_option("--muonCollectionTag",
250  help="If empty, use trajectories. If not empty, it's InputTag of muons collection to use in tracker muons based approach, e.g., 'newmuons' or 'muons'",
251  type="string",
252  default="",
253  dest="muonCollectionTag")
254 parser.add_option("--maxDxy",
255  help="maximum track impact parameter with relation to beamline",
256  type="string",
257  default="1000.",
258  dest="maxDxy")
259 parser.add_option("--minNCrossedChambers",
260  help="minimum number of muon chambers that a track is required to cross",
261  type="string",
262  default="3",
263  dest="minNCrossedChambers")
264 parser.add_option("--extraPlots",
265  help="produce additional plots with geometry, reports differences, and corrections visulizations",
266  action="store_true",
267  dest="extraPlots")
268 
269 if len(sys.argv) < 5:
270  raise SystemError("Too few arguments.\n\n"+parser.format_help())
271 
272 DIRNAME = sys.argv[1]
273 ITERATIONS = int(sys.argv[2])
274 INITIALGEOM = sys.argv[3]
275 INPUTFILES = sys.argv[4]
276 
277 options, args = parser.parse_args(sys.argv[5:])
278 user_mail = options.user_mail
279 mapplots_ingeneral = options.mapplots
280 segdiffplots_ingeneral = options.segdiffplots
281 curvatureplots_ingeneral = options.curvatureplots
282 globaltag = options.globaltag
283 trackerconnect = options.trackerconnect
284 trackeralignment = options.trackeralignment
285 trackerAPEconnect = options.trackerAPEconnect
286 trackerAPE = options.trackerAPE
287 trackerBowsconnect = options.trackerBowsconnect
288 trackerBows = options.trackerBows
289 gprcdconnect = options.gprcdconnect
290 gprcd = options.gprcd
291 iscosmics = str(options.iscosmics)
292 station123params = options.station123params
293 station4params = options.station4params
294 cscparams = options.cscparams
295 muonCollectionTag = options.muonCollectionTag
296 minTrackPt = options.minTrackPt
297 maxTrackPt = options.maxTrackPt
298 minTrackP = options.minTrackP
299 maxTrackP = options.maxTrackP
300 maxDxy = options.maxDxy
301 minTrackerHits = str(options.minTrackerHits)
302 maxTrackerRedChi2 = options.maxTrackerRedChi2
303 minNCrossedChambers = options.minNCrossedChambers
304 allowTIDTEC = str(not options.notAllowTIDTEC)
305 twoBin = str(options.twoBin)
306 weightAlignment = str(options.weightAlignment)
307 minAlignmentHits = str(options.minAlignmentHits)
308 combineME11 = str(not options.notCombineME11)
309 maxEvents = options.maxEvents
310 skipEvents = options.skipEvents
311 validationLabel = options.validationLabel
312 maxResSlopeY = options.maxResSlopeY
313 theNSigma = options.motionPolicyNSigma
314 residualsModel = options.residualsModel
315 peakNSigma = options.peakNSigma
316 preFilter = not not options.preFilter
317 extraPlots = options.extraPlots
318 useResiduals = options.useResiduals
319 
320 
321 #print "check: ", allowTIDTEC, combineME11, preFilter
322 
323 doCleanUp = not options.noCleanUp
324 createMapNtuple = not not options.createMapNtuple
325 createAlignNtuple = not not options.createAlignNtuple
326 
327 doCSC = True
328 if options.noCSC: doCSC = False
329 doDT = True
330 if options.noDT: doDT = False
331 if options.noCSC and options.noDT:
332  print("cannot do --noCSC and --noDT at the same time!")
333  sys.exit()
334 
335 json_file = options.json
336 
337 fileNames=[]
338 fileNamesBlocks=[]
339 execfile(INPUTFILES)
340 njobs = options.subjobs
341 if (options.inputInBlocks):
342  njobs = len(fileNamesBlocks)
343  if njobs==0:
344  print("while --inputInBlocks is specified, the INPUTFILES has no blocks!")
345  sys.exit()
346 
347 stepsize = int(math.ceil(1.*len(fileNames)/options.subjobs))
348 
349 pwd = str(os.getcwd())
350 
351 copytrackerdb = ""
352 if trackerconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerconnect[12:]
353 if trackerAPEconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerAPEconnect[12:]
354 if trackerBowsconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerBowsconnect[12:]
355 if gprcdconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % gprcdconnect[12:]
356 
357 
358 #####################################################################
359 # step 0: convert initial geometry to xml
360 INITIALXML = INITIALGEOM + '.xml'
361 if INITIALGEOM[-3:]=='.db':
362  INITIALXML = INITIALGEOM[:-3] + '.xml'
363 print("Converting",INITIALGEOM,"to",INITIALXML," ...will be done in several seconds...")
364 print("./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %s %s --gprcdconnect %s --gprcd %s" % (INITIALGEOM,INITIALXML,gprcdconnect,gprcd))
365 exit_code = os.system("./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %s %s --gprcdconnect %s --gprcd %s" % (INITIALGEOM,INITIALXML,gprcdconnect,gprcd))
366 if exit_code>0:
367  print("problem: conversion exited with code:", exit_code)
368  sys.exit()
369 
370 #####################################################################
371 
372 def writeGatherCfg(fname, my_vars):
373  file(fname, "w").write("""#/bin/sh
374 # %(commandline)s
375 
376 export ALIGNMENT_CAFDIR=`pwd`
377 
378 cd %(pwd)s
379 eval `scramv1 run -sh`
380 export ALIGNMENT_AFSDIR=`pwd`
381 
382 export ALIGNMENT_INPUTFILES='%(inputfiles)s'
383 export ALIGNMENT_ITERATION=%(iteration)d
384 export ALIGNMENT_JOBNUMBER=%(jobnumber)d
385 export ALIGNMENT_MAPPLOTS=%(mapplots)s
386 export ALIGNMENT_SEGDIFFPLOTS=%(segdiffplots)s
387 export ALIGNMENT_CURVATUREPLOTS=%(curvatureplots)s
388 export ALIGNMENT_GLOBALTAG=%(globaltag)s
389 export ALIGNMENT_INPUTDB=%(inputdb)s
390 export ALIGNMENT_TRACKERCONNECT=%(trackerconnect)s
391 export ALIGNMENT_TRACKERALIGNMENT=%(trackeralignment)s
392 export ALIGNMENT_TRACKERAPECONNECT=%(trackerAPEconnect)s
393 export ALIGNMENT_TRACKERAPE=%(trackerAPE)s
394 export ALIGNMENT_TRACKERBOWSCONNECT=%(trackerBowsconnect)s
395 export ALIGNMENT_TRACKERBOWS=%(trackerBows)s
396 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
397 export ALIGNMENT_GPRCD=%(gprcd)s
398 export ALIGNMENT_ISCOSMICS=%(iscosmics)s
399 export ALIGNMENT_STATION123PARAMS=%(station123params)s
400 export ALIGNMENT_STATION4PARAMS=%(station4params)s
401 export ALIGNMENT_CSCPARAMS=%(cscparams)s
402 export ALIGNMENT_MUONCOLLECTIONTAG=%(muonCollectionTag)s
403 export ALIGNMENT_MINTRACKPT=%(minTrackPt)s
404 export ALIGNMENT_MAXTRACKPT=%(maxTrackPt)s
405 export ALIGNMENT_MINTRACKP=%(minTrackP)s
406 export ALIGNMENT_MAXTRACKP=%(maxTrackP)s
407 export ALIGNMENT_MAXDXY=%(maxDxy)s
408 export ALIGNMENT_MINTRACKERHITS=%(minTrackerHits)s
409 export ALIGNMENT_MAXTRACKERREDCHI2=%(maxTrackerRedChi2)s
410 export ALIGNMENT_MINNCROSSEDCHAMBERS=%(minNCrossedChambers)s
411 export ALIGNMENT_ALLOWTIDTEC=%(allowTIDTEC)s
412 export ALIGNMENT_TWOBIN=%(twoBin)s
413 export ALIGNMENT_WEIGHTALIGNMENT=%(weightAlignment)s
414 export ALIGNMENT_MINALIGNMENTHITS=%(minAlignmentHits)s
415 export ALIGNMENT_COMBINEME11=%(combineME11)s
416 export ALIGNMENT_MAXEVENTS=%(maxEvents)s
417 export ALIGNMENT_SKIPEVENTS=%(skipEvents)s
418 export ALIGNMENT_MAXRESSLOPEY=%(maxResSlopeY)s
419 export ALIGNMENT_DO_DT=%(doDT)s
420 export ALIGNMENT_DO_CSC=%(doCSC)s
421 export ALIGNMENT_JSON=%(json_file)s
422 export ALIGNMENT_CREATEMAPNTUPLE=%(createMapNtuple)s
423 #export ALIGNMENT_CREATEALIGNNTUPLE=%(createAlignNtuple)s
424 export ALIGNMENT_PREFILTER=%(preFilter)s
425 
426 
427 if [ \"zzz$ALIGNMENT_JSON\" != \"zzz\" ]; then
428  cp -f $ALIGNMENT_JSON $ALIGNMENT_CAFDIR/
429 fi
430 
431 cp -f %(directory)sgather_cfg.py %(inputdbdir)s%(inputdb)s %(copytrackerdb)s $ALIGNMENT_CAFDIR/
432 cd $ALIGNMENT_CAFDIR/
433 ls -l
434 cmsRun gather_cfg.py
435 ls -l
436 cp -f *.tmp %(copyplots)s $ALIGNMENT_AFSDIR/%(directory)s
437 """ % my_vars)
438 
439 #####################################################################
440 
441 def writeAlignCfg(fname, my_vars):
442  file("%salign.sh" % directory, "w").write("""#!/bin/sh
443 # %(commandline)s
444 
445 export ALIGNMENT_CAFDIR=`pwd`
446 
447 cd %(pwd)s
448 eval `scramv1 run -sh`
449 export ALIGNMENT_AFSDIR=`pwd`
450 export ALIGNMENT_INPUTDB=%(inputdb)s
451 export ALIGNMENT_ITERATION=%(iteration)d
452 export ALIGNMENT_GLOBALTAG=%(globaltag)s
453 export ALIGNMENT_TRACKERCONNECT=%(trackerconnect)s
454 export ALIGNMENT_TRACKERALIGNMENT=%(trackeralignment)s
455 export ALIGNMENT_TRACKERAPECONNECT=%(trackerAPEconnect)s
456 export ALIGNMENT_TRACKERAPE=%(trackerAPE)s
457 export ALIGNMENT_TRACKERBOWSCONNECT=%(trackerBowsconnect)s
458 export ALIGNMENT_TRACKERBOWS=%(trackerBows)s
459 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
460 export ALIGNMENT_GPRCD=%(gprcd)s
461 export ALIGNMENT_ISCOSMICS=%(iscosmics)s
462 export ALIGNMENT_STATION123PARAMS=%(station123params)s
463 export ALIGNMENT_STATION4PARAMS=%(station4params)s
464 export ALIGNMENT_CSCPARAMS=%(cscparams)s
465 export ALIGNMENT_MINTRACKPT=%(minTrackPt)s
466 export ALIGNMENT_MAXTRACKPT=%(maxTrackPt)s
467 export ALIGNMENT_MINTRACKP=%(minTrackP)s
468 export ALIGNMENT_MAXTRACKP=%(maxTrackP)s
469 export ALIGNMENT_MINTRACKERHITS=%(minTrackerHits)s
470 export ALIGNMENT_MAXTRACKERREDCHI2=%(maxTrackerRedChi2)s
471 export ALIGNMENT_ALLOWTIDTEC=%(allowTIDTEC)s
472 export ALIGNMENT_TWOBIN=%(twoBin)s
473 export ALIGNMENT_WEIGHTALIGNMENT=%(weightAlignment)s
474 export ALIGNMENT_MINALIGNMENTHITS=%(minAlignmentHits)s
475 export ALIGNMENT_COMBINEME11=%(combineME11)s
476 export ALIGNMENT_MAXRESSLOPEY=%(maxResSlopeY)s
477 export ALIGNMENT_CLEANUP=%(doCleanUp)s
478 export ALIGNMENT_CREATEALIGNNTUPLE=%(createAlignNtuple)s
479 export ALIGNMENT_RESIDUALSMODEL=%(residualsModel)s
480 export ALIGNMENT_PEAKNSIGMA=%(peakNSigma)s
481 export ALIGNMENT_USERESIDUALS=%(useResiduals)s
482 
483 cp -f %(directory)salign_cfg.py %(inputdbdir)s%(inputdb)s %(directory)s*.tmp %(copytrackerdb)s $ALIGNMENT_CAFDIR/
484 
485 export ALIGNMENT_PLOTTINGTMP=`find %(directory)splotting0*.root -maxdepth 1 -size +0 -print 2> /dev/null`
486 
487 # if it's 1st or last iteration, combine _plotting.root files into one:
488 if [ \"$ALIGNMENT_ITERATION\" != \"111\" ] || [ \"$ALIGNMENT_ITERATION\" == \"%(ITERATIONS)s\" ]; then
489  #nfiles=$(ls %(directory)splotting0*.root 2> /dev/null | wc -l)
490  if [ \"zzz$ALIGNMENT_PLOTTINGTMP\" != \"zzz\" ]; then
491  hadd -f1 %(directory)s%(director)s_plotting.root %(directory)splotting0*.root
492  #if [ $? == 0 ] && [ \"$ALIGNMENT_CLEANUP\" == \"True\" ]; then rm %(directory)splotting0*.root; fi
493  fi
494 fi
495 
496 if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ \"zzz$ALIGNMENT_PLOTTINGTMP\" != \"zzz\" ]; then
497  rm $ALIGNMENT_PLOTTINGTMP
498 fi
499 
500 cd $ALIGNMENT_CAFDIR/
501 export ALIGNMENT_ALIGNMENTTMP=`find alignment*.tmp -maxdepth 1 -size +1k -print 2> /dev/null`
502 ls -l
503 
504 cmsRun align_cfg.py
505 cp -f MuonAlignmentFromReference_report.py $ALIGNMENT_AFSDIR/%(directory)s%(director)s_report.py
506 cp -f MuonAlignmentFromReference_outputdb.db $ALIGNMENT_AFSDIR/%(directory)s%(director)s.db
507 cp -f MuonAlignmentFromReference_plotting.root $ALIGNMENT_AFSDIR/%(directory)s%(director)s.root
508 
509 cd $ALIGNMENT_AFSDIR
510 ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s.db %(directory)s%(director)s.xml --noLayers --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
511 
512 export ALIGNMENT_ALIGNMENTTMP=`find %(directory)salignment*.tmp -maxdepth 1 -size +1k -print 2> /dev/null`
513 if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ \"zzz$ALIGNMENT_ALIGNMENTTMP\" != \"zzz\" ]; then
514  rm $ALIGNMENT_ALIGNMENTTMP
515  echo " "
516 fi
517 
518 # if it's not 1st or last iteration, do some clean up:
519 if [ \"$ALIGNMENT_ITERATION\" != \"1\" ] && [ \"$ALIGNMENT_ITERATION\" != \"%(ITERATIONS)s\" ]; then
520  if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ -e %(directory)s%(director)s.root ]; then
521  rm %(directory)s%(director)s.root
522  fi
523 fi
524 
525 # if it's last iteration, apply chamber motion policy
526 if [ \"$ALIGNMENT_ITERATION\" == \"%(ITERATIONS)s\" ]; then
527  # convert this iteration's geometry into detailed xml
528  ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s.db %(directory)s%(director)s_extra.xml --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
529  # perform motion policy
530  ./Alignment/MuonAlignmentAlgorithms/scripts/motionPolicyChamber.py \
531  %(INITIALXML)s %(directory)s%(director)s_extra.xml \
532  %(directory)s%(director)s_report.py \
533  %(directory)s%(director)s_final.xml \
534  --nsigma %(theNSigma)s
535  # convert the resulting xml into the final sqlite geometry
536  ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s_final.xml %(directory)s%(director)s_final.db --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
537 fi
538 
539 """ % my_vars)
540 
541 #####################################################################
542 
543 def writeValidationCfg(fname, my_vars):
544  file(fname, "w").write("""#!/bin/sh
545 # %(commandline)s
546 
547 export ALIGNMENT_CAFDIR=`pwd`
548 mkdir files
549 mkdir out
550 
551 cd %(pwd)s
552 eval `scramv1 run -sh`
553 ALIGNMENT_AFSDIR=`pwd`
554 ALIGNMENT_ITERATION=%(iteration)d
555 ALIGNMENT_MAPPLOTS=None
556 ALIGNMENT_SEGDIFFPLOTS=None
557 ALIGNMENT_CURVATUREPLOTS=None
558 ALIGNMENT_EXTRAPLOTS=%(extraPlots)s
559 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
560 export ALIGNMENT_GPRCD=%(gprcd)s
561 export ALIGNMENT_DO_DT=%(doDT)s
562 export ALIGNMENT_DO_CSC=%(doCSC)s
563 
564 
565 # copy the scripts to CAFDIR
566 cd Alignment/MuonAlignmentAlgorithms/scripts/
567 cp -f plotscripts.py $ALIGNMENT_CAFDIR/
568 cp -f mutypes.py $ALIGNMENT_CAFDIR/
569 cp -f alignmentValidation.py $ALIGNMENT_CAFDIR/
570 cp -f phiedges_fitfunctions.C $ALIGNMENT_CAFDIR/
571 cp -f createTree.py $ALIGNMENT_CAFDIR/
572 cp -f signConventions.py $ALIGNMENT_CAFDIR/
573 cp -f convertSQLiteXML.py $ALIGNMENT_CAFDIR/
574 cp -f wrapperExtraPlots.sh $ALIGNMENT_CAFDIR/
575 cd -
576 cp Alignment/MuonAlignmentAlgorithms/test/browser/tree* $ALIGNMENT_CAFDIR/out/
577 
578 # copy the results to CAFDIR
579 cp -f %(directory1)s%(director1)s_report.py $ALIGNMENT_CAFDIR/files/
580 cp -f %(directory)s%(director)s_report.py $ALIGNMENT_CAFDIR/files/
581 cp -f %(directory1)s%(director1)s.root $ALIGNMENT_CAFDIR/files/
582 cp -f %(directory)s%(director)s.root $ALIGNMENT_CAFDIR/files/
583 if [ -e %(directory1)s%(director1)s_plotting.root ] && [ -e %(directory)s%(director)s_plotting.root ]; then
584  cp -f %(directory1)s%(director1)s_plotting.root $ALIGNMENT_CAFDIR/files/
585  cp -f %(directory)s%(director)s_plotting.root $ALIGNMENT_CAFDIR/files/
586  ALIGNMENT_MAPPLOTS=%(mapplots)s
587  ALIGNMENT_SEGDIFFPLOTS=%(segdiffplots)s
588  ALIGNMENT_CURVATUREPLOTS=%(curvatureplots)s
589 fi
590 
591 dtcsc=""
592 if [ $ALIGNMENT_DO_DT == \"True\" ]; then
593  dtcsc="--dt"
594 fi
595 if [ $ALIGNMENT_DO_CSC == \"True\" ]; then
596  dtcsc="${dtcsc} --csc"
597 fi
598 
599 
600 cd $ALIGNMENT_CAFDIR/
601 echo \" ### Start running ###\"
602 date
603 
604 # do fits and median plots first
605 ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out --createDirSructure --dt --csc --fit --median
606 
607 if [ $ALIGNMENT_MAPPLOTS == \"True\" ]; then
608  ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out $dtcsc --map
609 fi
610 
611 if [ $ALIGNMENT_SEGDIFFPLOTS == \"True\" ]; then
612  ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out $dtcsc --segdiff
613 fi
614 
615 if [ $ALIGNMENT_CURVATUREPLOTS == \"True\" ]; then
616  ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out $dtcsc --curvature
617 fi
618 
619 if [ $ALIGNMENT_EXTRAPLOTS == \"True\" ]; then
620  if [ \"zzz%(copytrackerdb)s\" != \"zzz\" ]; then
621  cp -f $ALIGNMENT_AFSDIR/%(copytrackerdb)s $ALIGNMENT_CAFDIR/
622  fi
623  cp $ALIGNMENT_AFSDIR/inertGlobalPositionRcd.db .
624  ./convertSQLiteXML.py $ALIGNMENT_AFSDIR/%(INITIALGEOM)s g0.xml --noLayers --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
625  ./wrapperExtraPlots.sh -n $ALIGNMENT_ITERATION -i $ALIGNMENT_AFSDIR -0 g0.xml -z -w %(station123params)s %(dir_no_)s
626  mkdir out/extra
627  cd %(dir_no_)s
628  mv MB ../out/extra/
629  mv ME ../out/extra/
630  cd -
631 fi
632 
633 # run simple diagnostic
634 ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out --dt --csc --diagnostic
635 
636 # fill the tree browser structure:
637 ./createTree.py -i $ALIGNMENT_CAFDIR/out
638 
639 timestamp=`date \"+%%y-%%m-%%d %%H:%%M:%%S\"`
640 echo \"%(validationLabel)s.plots (${timestamp})\" > out/label.txt
641 
642 ls -l out/
643 timestamp=`date +%%Y%%m%%d%%H%%M%%S`
644 tar czf %(validationLabel)s_${timestamp}.tgz out
645 cp -f %(validationLabel)s_${timestamp}.tgz $ALIGNMENT_AFSDIR/
646 
647 """ % my_vars)
648 
649 
650 #####################################################################
651 
652 #SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS = True
653 SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS = False
654 
655 bsubfile = ["#!/bin/sh", ""]
656 bsubnames = []
657 last_align = None
658 directory = ""
659 
660 for iteration in range(1, ITERATIONS+1):
661  if iteration == 1:
662  inputdb = INITIALGEOM
663  inputdbdir = directory[:]
664  else:
665  inputdb = director + ".db"
666  inputdbdir = directory[:]
667 
668  directory = "%s%02d/" % (DIRNAME, iteration)
669  director = directory[:-1]
670 
671  dir_no_ = DIRNAME
672  if DIRNAME[-1]=='_': dir_no_ = DIRNAME[:-1]
673 
674  os.system("rm -rf %s; mkdir %s" % (directory, directory))
675  os.system("cp Alignment/MuonAlignmentAlgorithms/python/gather_cfg.py %s" % directory)
676  os.system("cp Alignment/MuonAlignmentAlgorithms/python/align_cfg.py %s" % directory)
677 
678  bsubfile.append("cd %s" % directory)
679 
680  mapplots = False
681  if mapplots_ingeneral and (iteration == 1 or iteration == 3 or iteration == 5 or iteration == 7 or iteration == 9 or iteration == ITERATIONS): mapplots = True
682  segdiffplots = False
683  if segdiffplots_ingeneral and (iteration == 1 or iteration == ITERATIONS): segdiffplots = True
684  curvatureplots = False
685  if curvatureplots_ingeneral and (iteration == 1 or iteration == ITERATIONS): curvatureplots = True
686 
687  ### gather.sh runners for njobs
688  for jobnumber in range(njobs):
689  if not options.inputInBlocks:
690  inputfiles = " ".join(fileNames[jobnumber*stepsize:(jobnumber+1)*stepsize])
691  else:
692  inputfiles = " ".join(fileNamesBlocks[jobnumber])
693 
694  if mapplots or segdiffplots or curvatureplots: copyplots = "plotting*.root"
695  else: copyplots = ""
696 
697  if len(inputfiles) > 0:
698  gather_fileName = "%sgather%03d.sh" % (directory, jobnumber)
699  writeGatherCfg(gather_fileName, vars())
700  os.system("chmod +x %s" % gather_fileName)
701  bsubfile.append("echo %sgather%03d.sh" % (directory, jobnumber))
702 
703  if last_align is None: waiter = ""
704  else: waiter = "-w \"ended(%s)\"" % last_align
705  if options.big: queue = "cmscaf1nd"
706  else: queue = "cmscaf1nh"
707 
708  bsubfile.append("bsub -R \"type==SLC6_64\" -q %s -J \"%s_gather%03d\" -u youremail.tamu.edu %s gather%03d.sh" % (queue, director, jobnumber, waiter, jobnumber))
709 
710  bsubnames.append("ended(%s_gather%03d)" % (director, jobnumber))
711 
712 
713  ### align.sh
714  if SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS:
715  if ( iteration == 1 or iteration == 3 or iteration == 5 or iteration == 7 or iteration == 9):
716  tmp = station123params, station123params, useResiduals
717  station123params, station123params, useResiduals = "000010", "000010", "0010"
718  writeAlignCfg("%salign.sh" % directory, vars())
719  station123params, station123params, useResiduals = tmp
720  elif ( iteration == 2 or iteration == 4 or iteration == 6 or iteration == 8 or iteration == 10):
721  tmp = station123params, station123params, useResiduals
722  station123params, station123params, useResiduals = "110001", "100001", "1100"
723  writeAlignCfg("%salign.sh" % directory, vars())
724  station123params, station123params, useResiduals = tmp
725  else:
726  writeAlignCfg("%salign.sh" % directory, vars())
727 
728  os.system("chmod +x %salign.sh" % directory)
729 
730  bsubfile.append("echo %salign.sh" % directory)
731  if user_mail: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_align\" -u %s -w \"%s\" align.sh" % (director, user_mail, " && ".join(bsubnames)))
732  else: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_align\" -w \"%s\" align.sh" % (director, " && ".join(bsubnames)))
733 
734  #bsubfile.append("cd ..")
735  bsubnames = []
736  last_align = "%s_align" % director
737 
738 
739  ### after the last iteration (optionally) do diagnostics run
740  if len(validationLabel) and iteration == ITERATIONS:
741  # do we have plotting files created?
742  directory1 = "%s01/" % DIRNAME
743  director1 = directory1[:-1]
744 
745  writeValidationCfg("%svalidation.sh" % directory, vars())
746  os.system("chmod +x %svalidation.sh" % directory)
747 
748  bsubfile.append("echo %svalidation.sh" % directory)
749  if user_mail: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_validation\" -u %s -w \"ended(%s)\" validation.sh" % (director, user_mail, last_align))
750  else: bsubfile.append("bsub -R \"type==SLC6_64\" -q cmscaf1nd -J \"%s_validation\" -w \"ended(%s)\" validation.sh" % (director, last_align))
751 
752  bsubfile.append("cd ..")
753  bsubfile.append("")
754 
755 
756 file(options.submitJobs, "w").write("\n".join(bsubfile))
757 os.system("chmod +x %s" % options.submitJobs)
createJobs.writeAlignCfg
def writeAlignCfg(fname, my_vars)
Definition: createJobs.py:441
edm::print
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
createJobs.writeGatherCfg
def writeGatherCfg(fname, my_vars)
Definition: createJobs.py:372
spr::find
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:20
createJobs.writeValidationCfg
def writeValidationCfg(fname, my_vars)
Definition: createJobs.py:543
FrontierConditions_GlobalTag_cff.file
file
Definition: FrontierConditions_GlobalTag_cff.py:13
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
TriggerAnalyzer.write
def write(self, setup)
Definition: TriggerAnalyzer.py:98
str
#define str(s)
Definition: TestProcessor.cc:48
Generated for CMSSW Reference Manual by doxygen 1.8.11