CMS 3D CMS Logo

/data/refman/pasoursint/CMSSW_5_3_3/src/Alignment/MuonAlignmentAlgorithms/scripts/createJobs.py

Go to the documentation of this file.
00001 #! /usr/bin/env python
00002 
00003 import os, sys, optparse, math
00004 
00005 copyargs = sys.argv[:]
00006 for i in range(len(copyargs)):
00007     if copyargs[i] == "":
00008         copyargs[i] = "\"\""
00009     if copyargs[i].find(" ") != -1:
00010         copyargs[i] = "\"%s\"" % copyargs[i]
00011 commandline = " ".join(copyargs)
00012 
00013 prog = sys.argv[0]
00014 
00015 usage = """./%(prog)s DIRNAME ITERATIONS INITIALGEOM INPUTFILES [options]
00016 
00017 Creates (overwrites) a directory for each of the iterations and creates (overwrites)
00018 submitJobs.sh with the submission sequence and dependencies.
00019 
00020 DIRNAME        directories will be named DIRNAME01, DIRNAME02, etc.
00021 ITERATIONS     number of iterations
00022 INITIALGEOM    SQLite file containing muon geometry with tag names
00023                DTAlignmentRcd, DTAlignmentErrorRcd, CSCAlignmentRcd, CSCAlignmentErrorRcd
00024 INPUTFILES     Python file defining 'fileNames', a list of input files as
00025                strings (create with findQualityFiles.py)""" % vars()
00026 
00027 parser = optparse.OptionParser(usage)
00028 parser.add_option("-j", "--jobs",
00029                    help="approximate number of \"gather\" subjobs",
00030                    type="int",
00031                    default=50,
00032                    dest="subjobs")
00033 parser.add_option("-s", "--submitJobs",
00034                    help="alternate name of submitJobs.sh script (please include .sh extension); a file with this name will be OVERWRITTEN",
00035                    type="string",
00036                    default="submitJobs.sh",
00037                    dest="submitJobs")
00038 parser.add_option("-b", "--big",
00039                   help="if invoked, subjobs will also be run on cmscaf1nd",
00040                   action="store_true",
00041                   dest="big")
00042 parser.add_option("-u", "--user_mail",
00043                   help="if invoked, send mail to a specified email destination. If \"-u\" is not present, the default destination LSB_MAILTO in lsf.conf will be used",
00044                   type="string",
00045                   dest="user_mail")
00046 parser.add_option("--mapplots",
00047                   help="if invoked, draw \"map plots\"",
00048                   action="store_true",
00049                   dest="mapplots")
00050 parser.add_option("--segdiffplots",
00051                   help="if invoked, draw \"segment-difference plots\"",
00052                   action="store_true",
00053                   dest="segdiffplots")
00054 parser.add_option("--curvatureplots",
00055                   help="if invoked, draw \"curvature plots\"",
00056                   action="store_true",
00057                   dest="curvatureplots")
00058 parser.add_option("--globalTag",
00059                   help="GlobalTag for alignment/calibration conditions (typically all conditions except muon and tracker alignment)",
00060                   type="string",
00061                   default="CRAFT0831X_V1::All",
00062                   dest="globaltag")
00063 parser.add_option("--trackerconnect",
00064                   help="connect string for tracker alignment (frontier://FrontierProd/CMS_COND_310X_ALIGN or sqlite_file:...)",
00065                   type="string",
00066                   default="",
00067                   dest="trackerconnect")
00068 parser.add_option("--trackeralignment",
00069                   help="name of TrackerAlignmentRcd tag",
00070                   type="string",
00071                   default="Alignments",
00072                   dest="trackeralignment")
00073 parser.add_option("--trackerAPEconnect",
00074                   help="connect string for tracker APEs (frontier://... or sqlite_file:...)",
00075                   type="string",
00076                   default="",
00077                   dest="trackerAPEconnect")
00078 parser.add_option("--trackerAPE",
00079                   help="name of TrackerAlignmentErrorRcd tag (tracker APEs)",
00080                   type="string",
00081                   default="AlignmentErrors",
00082                   dest="trackerAPE")
00083 parser.add_option("--trackerBowsconnect",
00084                   help="connect string for tracker Surface Deformations (frontier://... or sqlite_file:...)",
00085                   type="string",
00086                   default="",
00087                   dest="trackerBowsconnect")
00088 parser.add_option("--trackerBows",
00089                   help="name of TrackerSurfaceDeformationRcd tag",
00090                   type="string",
00091                   default="TrackerSurfaceDeformations",
00092                   dest="trackerBows")
00093 parser.add_option("--gprcdconnect",
00094                   help="connect string for GlobalPositionRcd (frontier://... or sqlite_file:...)",
00095                   type="string",
00096                   default="",
00097                   dest="gprcdconnect")
00098 parser.add_option("--gprcd",
00099                   help="name of GlobalPositionRcd tag",
00100                   type="string",
00101                   default="GlobalPosition",
00102                   dest="gprcd")
00103 parser.add_option("--iscosmics",
00104                   help="if invoked, use cosmic track refitter instead of the standard one",
00105                   action="store_true",
00106                   dest="iscosmics")
00107 parser.add_option("--station123params",
00108                   help="alignable parameters for DT stations 1, 2, 3 (see SWGuideAlignmentAlgorithms#Selection_of_what_to_align)",
00109                   type="string",
00110                   default="111111",
00111                   dest="station123params")
00112 parser.add_option("--station4params",
00113                   help="alignable parameters for DT station 4",
00114                   type="string",
00115                   default="100011",
00116                   dest="station4params")
00117 parser.add_option("--cscparams",
00118                   help="alignable parameters for CSC chambers",
00119                   type="string",
00120                   default="100011",
00121                   dest="cscparams")
00122 parser.add_option("--minTrackPt",
00123                   help="minimum allowed track transverse momentum (in GeV)",
00124                   type="string",
00125                   default="0",
00126                   dest="minTrackPt")
00127 parser.add_option("--maxTrackPt",
00128                   help="maximum allowed track transverse momentum (in GeV)",
00129                   type="string",
00130                   default="1000",
00131                   dest="maxTrackPt")
00132 parser.add_option("--minTrackP",
00133                   help="minimum allowed track momentum (in GeV)",
00134                   type="string",
00135                   default="0",
00136                   dest="minTrackP")
00137 parser.add_option("--maxTrackP",
00138                   help="maximum allowed track momentum (in GeV)",
00139                   type="string",
00140                   default="10000",
00141                   dest="maxTrackP")
00142 parser.add_option("--minTrackerHits",
00143                   help="minimum number of tracker hits",
00144                   type="int",
00145                   default=15,
00146                   dest="minTrackerHits")
00147 parser.add_option("--maxTrackerRedChi2",
00148                   help="maximum tracker chi^2 per degrees of freedom",
00149                   type="string",
00150                   default="10",
00151                   dest="maxTrackerRedChi2")
00152 parser.add_option("--notAllowTIDTEC",
00153                   help="if invoked, do not allow tracks that pass through the tracker's TID||TEC region (not recommended)",
00154                   action="store_true",
00155                   dest="notAllowTIDTEC")
00156 parser.add_option("--twoBin",
00157                   help="if invoked, apply the \"two-bin method\" to control charge-antisymmetric errors",
00158                   action="store_true",
00159                   dest="twoBin")
00160 parser.add_option("--weightAlignment",
00161                   help="if invoked, segments will be weighted by ndf/chi^2 in the alignment",
00162                   action="store_true",
00163                   dest="weightAlignment")
00164 parser.add_option("--minAlignmentSegments",
00165                   help="minimum number of segments required to align a chamber",
00166                   type="int",
00167                   default=5,
00168                   dest="minAlignmentHits")
00169 parser.add_option("--notCombineME11",
00170                   help="if invoced, treat ME1/1a and ME1/1b as separate objects",
00171                   action="store_true",
00172                   dest="notCombineME11")
00173 parser.add_option("--maxEvents",
00174                   help="maximum number of events",
00175                   type="string",
00176                   default="-1",
00177                   dest="maxEvents")
00178 parser.add_option("--skipEvents",
00179                   help="number of events to be skipped",
00180                   type="string",
00181                   default="0",
00182                   dest="skipEvents")
00183 parser.add_option("--validationLabel",
00184                   help="if given nonempty string RUNLABEL, diagnostics and creation of plots will be run in the end of the last iteration; the RUNLABEL will be used to mark a run; the results will be put into a RUNLABEL_DATESTAMP.tgz tarball",
00185                   type="string",
00186                   default="",
00187                   dest="validationLabel")
00188 parser.add_option("--maxResSlopeY",
00189                   help="maximum residual slope y component",
00190                   type="string",
00191                   default="10",
00192                   dest="maxResSlopeY")
00193 parser.add_option("--motionPolicyNSigma",
00194                   help="minimum nsigma(deltax) position displacement in order to move a chamber for the final alignment result; default NSIGMA=3",
00195                   type="int",
00196                   default=3,
00197                   dest="motionPolicyNSigma")
00198 parser.add_option("--noCleanUp",
00199                   help="if invoked, temporary plotting???.root and *.tmp files would not be removed at the end of each align job",
00200                   action="store_true",
00201                   dest="noCleanUp")
00202 parser.add_option("--noCSC",
00203                   help="if invoked, CSC endcap chambers would not be processed",
00204                   action="store_true",
00205                   dest="noCSC")
00206 parser.add_option("--noDT",
00207                   help="if invoked, DT barrel chambers would not be processed",
00208                   action="store_true",
00209                   dest="noDT")
00210 parser.add_option("--createMapNtuple",
00211                   help="if invoked while mapplots are switched on, a special ntuple would be created",
00212                   action="store_true",
00213                   dest="createMapNtuple")
00214 parser.add_option("--inputInBlocks",
00215                   help="if invoked, assume that INPUTFILES provides a list of files already groupped into job blocks, -j has no effect in that case",
00216                   action="store_true",
00217                   dest="inputInBlocks")
00218 parser.add_option("--json",
00219                   help="If present with JSON file as argument, use JSON file for good lumi mask. "+\
00220                   "The latest JSON file is available at /afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions11/7TeV/Prompt/",
00221                   type="string",
00222                   default="",
00223                   dest="json")
00224 parser.add_option("--createAlignNtuple",
00225                   help="if invoked, debug ntuples with residuals would be created during gather jobs",
00226                   action="store_true",
00227                   dest="createAlignNtuple")
00228 parser.add_option("--residualsModel",
00229                   help="functional residuals model. Possible vaslues: pureGaussian2D (default), pureGaussian, GaussPowerTails, ROOTVoigt, powerLawTails",
00230                   type="string",
00231                   default="pureGaussian2D",
00232                   dest="residualsModel")
00233 parser.add_option("--useResiduals",
00234                   help="select residuals to use, possible values: 1111, 1110, 1100, 1010, 0010 that correspond to x y dxdz dydz residuals",
00235                   type="string",
00236                   default="1110",
00237                   dest="useResiduals")
00238 parser.add_option("--peakNSigma",
00239                   help="if >0, only residuals peaks within n-sigma multidimentional ellipsoid would be considered in the alignment fit",
00240                   type="string",
00241                   default="-1.",
00242                   dest="peakNSigma")
00243 parser.add_option("--preFilter",
00244                   help="if invoked, MuonAlignmentPreFilter module would be invoked in the Path's beginning. Can significantly speed up gather jobs.",
00245                   action="store_true",
00246                   dest="preFilter")
00247 parser.add_option("--muonCollectionTag",
00248                   help="If empty, use trajectories. If not empty, it's InputTag of muons collection to use in tracker muons based approach, e.g., 'newmuons' or 'muons'",
00249                   type="string",
00250                   default="",
00251                   dest="muonCollectionTag")
00252 parser.add_option("--maxDxy",
00253                   help="maximum track impact parameter with relation to beamline",
00254                   type="string",
00255                   default="1000.",
00256                   dest="maxDxy")
00257 parser.add_option("--minNCrossedChambers",
00258                   help="minimum number of muon chambers that a track is required to cross",
00259                   type="string",
00260                   default="3",
00261                   dest="minNCrossedChambers")
00262 parser.add_option("--extraPlots",
00263                   help="produce additional plots with geometry, reports differences, and corrections visulizations",
00264                   action="store_true",
00265                   dest="extraPlots")
00266 
00267 if len(sys.argv) < 5:
00268     raise SystemError, "Too few arguments.\n\n"+parser.format_help()
00269 
00270 DIRNAME = sys.argv[1]
00271 ITERATIONS = int(sys.argv[2])
00272 INITIALGEOM = sys.argv[3]
00273 INPUTFILES = sys.argv[4]
00274 
00275 options, args = parser.parse_args(sys.argv[5:])
00276 user_mail = options.user_mail
00277 mapplots_ingeneral = options.mapplots
00278 segdiffplots_ingeneral = options.segdiffplots
00279 curvatureplots_ingeneral = options.curvatureplots
00280 globaltag = options.globaltag
00281 trackerconnect = options.trackerconnect
00282 trackeralignment = options.trackeralignment
00283 trackerAPEconnect = options.trackerAPEconnect
00284 trackerAPE = options.trackerAPE
00285 trackerBowsconnect = options.trackerBowsconnect
00286 trackerBows = options.trackerBows
00287 gprcdconnect = options.gprcdconnect
00288 gprcd = options.gprcd
00289 iscosmics = str(options.iscosmics)
00290 station123params = options.station123params
00291 station4params = options.station4params
00292 cscparams = options.cscparams
00293 muonCollectionTag = options.muonCollectionTag
00294 minTrackPt = options.minTrackPt
00295 maxTrackPt = options.maxTrackPt
00296 minTrackP = options.minTrackP
00297 maxTrackP = options.maxTrackP
00298 maxDxy = options.maxDxy
00299 minTrackerHits = str(options.minTrackerHits)
00300 maxTrackerRedChi2 = options.maxTrackerRedChi2
00301 minNCrossedChambers = options.minNCrossedChambers
00302 allowTIDTEC = str(not options.notAllowTIDTEC)
00303 twoBin = str(options.twoBin)
00304 weightAlignment = str(options.weightAlignment)
00305 minAlignmentHits = str(options.minAlignmentHits)
00306 combineME11 = str(not options.notCombineME11)
00307 maxEvents = options.maxEvents
00308 skipEvents = options.skipEvents
00309 validationLabel = options.validationLabel
00310 maxResSlopeY = options.maxResSlopeY
00311 theNSigma = options.motionPolicyNSigma
00312 residualsModel = options.residualsModel
00313 peakNSigma = options.peakNSigma
00314 preFilter = not not options.preFilter
00315 extraPlots = options.extraPlots
00316 useResiduals = options.useResiduals
00317 
00318 
00319 #print "check: ", allowTIDTEC, combineME11, preFilter
00320 
00321 doCleanUp = not options.noCleanUp
00322 createMapNtuple = not not options.createMapNtuple
00323 createAlignNtuple = not not options.createAlignNtuple
00324 
00325 doCSC = True
00326 if options.noCSC: doCSC = False
00327 doDT = True
00328 if options.noDT: doDT = False
00329 if options.noCSC and options.noDT:
00330   print "cannot do --noCSC and --noDT at the same time!"
00331   sys.exit()
00332 
00333 json_file = options.json
00334 
00335 fileNames=[]
00336 fileNamesBlocks=[]
00337 execfile(INPUTFILES)
00338 njobs = options.subjobs
00339 if (options.inputInBlocks):
00340   njobs = len(fileNamesBlocks)
00341   if njobs==0:
00342     print "while --inputInBlocks is specified, the INPUTFILES has no blocks!"
00343     sys.exit()
00344 
00345 stepsize = int(math.ceil(1.*len(fileNames)/options.subjobs))
00346 
00347 pwd = str(os.getcwdu())
00348 
00349 copytrackerdb = ""
00350 if trackerconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerconnect[12:]
00351 if trackerAPEconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerAPEconnect[12:]
00352 if trackerBowsconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % trackerBowsconnect[12:]
00353 if gprcdconnect[0:12] == "sqlite_file:": copytrackerdb += "%s " % gprcdconnect[12:]
00354 
00355 
00356 #####################################################################
00357 # step 0: convert initial geometry to xml
00358 INITIALXML = INITIALGEOM + '.xml'
00359 if INITIALGEOM[-3:]=='.db':
00360   INITIALXML = INITIALGEOM[:-3] + '.xml'
00361 print "Converting",INITIALGEOM,"to",INITIALXML," ...will be done in several seconds..."
00362 print "./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py  %s %s --gprcdconnect %s --gprcd %s" % (INITIALGEOM,INITIALXML,gprcdconnect,gprcd)
00363 exit_code = os.system("./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py  %s %s --gprcdconnect %s --gprcd %s" % (INITIALGEOM,INITIALXML,gprcdconnect,gprcd))
00364 if exit_code>0:
00365   print "problem: conversion exited with code:", exit_code
00366   sys.exit()
00367 
00368 #####################################################################
00369 
00370 def writeGatherCfg(fname, my_vars):
00371     file(fname, "w").write("""#/bin/sh
00372 # %(commandline)s
00373 
00374 export ALIGNMENT_CAFDIR=`pwd`
00375 
00376 cd %(pwd)s
00377 eval `scramv1 run -sh`
00378 export ALIGNMENT_AFSDIR=`pwd`
00379 
00380 export ALIGNMENT_INPUTFILES='%(inputfiles)s'
00381 export ALIGNMENT_ITERATION=%(iteration)d
00382 export ALIGNMENT_JOBNUMBER=%(jobnumber)d
00383 export ALIGNMENT_MAPPLOTS=%(mapplots)s
00384 export ALIGNMENT_SEGDIFFPLOTS=%(segdiffplots)s
00385 export ALIGNMENT_CURVATUREPLOTS=%(curvatureplots)s
00386 export ALIGNMENT_GLOBALTAG=%(globaltag)s
00387 export ALIGNMENT_INPUTDB=%(inputdb)s
00388 export ALIGNMENT_TRACKERCONNECT=%(trackerconnect)s
00389 export ALIGNMENT_TRACKERALIGNMENT=%(trackeralignment)s
00390 export ALIGNMENT_TRACKERAPECONNECT=%(trackerAPEconnect)s
00391 export ALIGNMENT_TRACKERAPE=%(trackerAPE)s
00392 export ALIGNMENT_TRACKERBOWSCONNECT=%(trackerBowsconnect)s
00393 export ALIGNMENT_TRACKERBOWS=%(trackerBows)s
00394 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
00395 export ALIGNMENT_GPRCD=%(gprcd)s
00396 export ALIGNMENT_ISCOSMICS=%(iscosmics)s
00397 export ALIGNMENT_STATION123PARAMS=%(station123params)s
00398 export ALIGNMENT_STATION4PARAMS=%(station4params)s
00399 export ALIGNMENT_CSCPARAMS=%(cscparams)s
00400 export ALIGNMENT_MUONCOLLECTIONTAG=%(muonCollectionTag)s
00401 export ALIGNMENT_MINTRACKPT=%(minTrackPt)s
00402 export ALIGNMENT_MAXTRACKPT=%(maxTrackPt)s
00403 export ALIGNMENT_MINTRACKP=%(minTrackP)s
00404 export ALIGNMENT_MAXTRACKP=%(maxTrackP)s
00405 export ALIGNMENT_MAXDXY=%(maxDxy)s
00406 export ALIGNMENT_MINTRACKERHITS=%(minTrackerHits)s
00407 export ALIGNMENT_MAXTRACKERREDCHI2=%(maxTrackerRedChi2)s
00408 export ALIGNMENT_MINNCROSSEDCHAMBERS=%(minNCrossedChambers)s
00409 export ALIGNMENT_ALLOWTIDTEC=%(allowTIDTEC)s
00410 export ALIGNMENT_TWOBIN=%(twoBin)s
00411 export ALIGNMENT_WEIGHTALIGNMENT=%(weightAlignment)s
00412 export ALIGNMENT_MINALIGNMENTHITS=%(minAlignmentHits)s
00413 export ALIGNMENT_COMBINEME11=%(combineME11)s
00414 export ALIGNMENT_MAXEVENTS=%(maxEvents)s
00415 export ALIGNMENT_SKIPEVENTS=%(skipEvents)s
00416 export ALIGNMENT_MAXRESSLOPEY=%(maxResSlopeY)s
00417 export ALIGNMENT_DO_DT=%(doDT)s
00418 export ALIGNMENT_DO_CSC=%(doCSC)s
00419 export ALIGNMENT_JSON=%(json_file)s
00420 export ALIGNMENT_CREATEMAPNTUPLE=%(createMapNtuple)s
00421 #export ALIGNMENT_CREATEALIGNNTUPLE=%(createAlignNtuple)s
00422 export ALIGNMENT_PREFILTER=%(preFilter)s
00423 
00424 
00425 if [ \"zzz$ALIGNMENT_JSON\" != \"zzz\" ]; then
00426   cp -f $ALIGNMENT_JSON $ALIGNMENT_CAFDIR/
00427 fi
00428 
00429 cp -f %(directory)sgather_cfg.py %(inputdbdir)s%(inputdb)s %(copytrackerdb)s $ALIGNMENT_CAFDIR/
00430 cd $ALIGNMENT_CAFDIR/
00431 ls -l
00432 cmsRun gather_cfg.py
00433 ls -l
00434 cp -f *.tmp %(copyplots)s $ALIGNMENT_AFSDIR/%(directory)s
00435 """ % my_vars)
00436 
00437 #####################################################################
00438 
00439 def writeAlignCfg(fname, my_vars):
00440     file("%salign.sh" % directory, "w").write("""#!/bin/sh
00441 # %(commandline)s
00442 
00443 export ALIGNMENT_CAFDIR=`pwd`
00444 
00445 cd %(pwd)s
00446 eval `scramv1 run -sh`
00447 export ALIGNMENT_AFSDIR=`pwd`
00448 export ALIGNMENT_INPUTDB=%(inputdb)s
00449 export ALIGNMENT_ITERATION=%(iteration)d
00450 export ALIGNMENT_GLOBALTAG=%(globaltag)s
00451 export ALIGNMENT_TRACKERCONNECT=%(trackerconnect)s
00452 export ALIGNMENT_TRACKERALIGNMENT=%(trackeralignment)s
00453 export ALIGNMENT_TRACKERAPECONNECT=%(trackerAPEconnect)s
00454 export ALIGNMENT_TRACKERAPE=%(trackerAPE)s
00455 export ALIGNMENT_TRACKERBOWSCONNECT=%(trackerBowsconnect)s
00456 export ALIGNMENT_TRACKERBOWS=%(trackerBows)s
00457 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
00458 export ALIGNMENT_GPRCD=%(gprcd)s
00459 export ALIGNMENT_ISCOSMICS=%(iscosmics)s
00460 export ALIGNMENT_STATION123PARAMS=%(station123params)s
00461 export ALIGNMENT_STATION4PARAMS=%(station4params)s
00462 export ALIGNMENT_CSCPARAMS=%(cscparams)s
00463 export ALIGNMENT_MINTRACKPT=%(minTrackPt)s
00464 export ALIGNMENT_MAXTRACKPT=%(maxTrackPt)s
00465 export ALIGNMENT_MINTRACKP=%(minTrackP)s
00466 export ALIGNMENT_MAXTRACKP=%(maxTrackP)s
00467 export ALIGNMENT_MINTRACKERHITS=%(minTrackerHits)s
00468 export ALIGNMENT_MAXTRACKERREDCHI2=%(maxTrackerRedChi2)s
00469 export ALIGNMENT_ALLOWTIDTEC=%(allowTIDTEC)s
00470 export ALIGNMENT_TWOBIN=%(twoBin)s
00471 export ALIGNMENT_WEIGHTALIGNMENT=%(weightAlignment)s
00472 export ALIGNMENT_MINALIGNMENTHITS=%(minAlignmentHits)s
00473 export ALIGNMENT_COMBINEME11=%(combineME11)s
00474 export ALIGNMENT_MAXRESSLOPEY=%(maxResSlopeY)s
00475 export ALIGNMENT_CLEANUP=%(doCleanUp)s
00476 export ALIGNMENT_CREATEALIGNNTUPLE=%(createAlignNtuple)s
00477 export ALIGNMENT_RESIDUALSMODEL=%(residualsModel)s
00478 export ALIGNMENT_PEAKNSIGMA=%(peakNSigma)s
00479 export ALIGNMENT_USERESIDUALS=%(useResiduals)s
00480 
00481 cp -f %(directory)salign_cfg.py %(inputdbdir)s%(inputdb)s %(directory)s*.tmp  %(copytrackerdb)s $ALIGNMENT_CAFDIR/
00482 
00483 export ALIGNMENT_PLOTTINGTMP=`ls %(directory)splotting0*.root 2> /dev/null`
00484 
00485 # if it's 1st or last iteration, combine _plotting.root files into one:
00486 if [ \"$ALIGNMENT_ITERATION\" != \"111\" ] || [ \"$ALIGNMENT_ITERATION\" == \"%(ITERATIONS)s\" ]; then
00487   #nfiles=$(ls %(directory)splotting0*.root 2> /dev/null | wc -l)
00488   if [ \"zzz$ALIGNMENT_PLOTTINGTMP\" != \"zzz\" ]; then
00489     hadd -f1 %(directory)s%(director)s_plotting.root %(directory)splotting0*.root
00490     #if [ $? == 0 ] && [ \"$ALIGNMENT_CLEANUP\" == \"True\" ]; then rm %(directory)splotting0*.root; fi
00491   fi
00492 fi
00493 
00494 if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ \"zzz$ALIGNMENT_PLOTTINGTMP\" != \"zzz\" ]; then
00495   rm $ALIGNMENT_PLOTTINGTMP
00496 fi
00497 
00498 cd $ALIGNMENT_CAFDIR/
00499 export ALIGNMENT_ALIGNMENTTMP=`ls alignment*.tmp 2> /dev/null`
00500 ls -l
00501 
00502 cmsRun align_cfg.py
00503 cp -f MuonAlignmentFromReference_report.py $ALIGNMENT_AFSDIR/%(directory)s%(director)s_report.py
00504 cp -f MuonAlignmentFromReference_outputdb.db $ALIGNMENT_AFSDIR/%(directory)s%(director)s.db
00505 cp -f MuonAlignmentFromReference_plotting.root $ALIGNMENT_AFSDIR/%(directory)s%(director)s.root
00506 
00507 cd $ALIGNMENT_AFSDIR
00508 ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s.db %(directory)s%(director)s.xml --noLayers --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
00509 
00510 export ALIGNMENT_ALIGNMENTTMP=`ls %(directory)salignment*.tmp 2> /dev/null`
00511 if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ \"zzz$ALIGNMENT_ALIGNMENTTMP\" != \"zzz\" ]; then
00512   rm $ALIGNMENT_ALIGNMENTTMP
00513   echo " "
00514 fi
00515 
00516 # if it's not 1st or last iteration, do some clean up:
00517 if [ \"$ALIGNMENT_ITERATION\" != \"1\" ] && [ \"$ALIGNMENT_ITERATION\" != \"%(ITERATIONS)s\" ]; then
00518   if [ \"$ALIGNMENT_CLEANUP\" == \"True\" ] && [ -e %(directory)s%(director)s.root ]; then
00519     rm %(directory)s%(director)s.root
00520   fi
00521 fi
00522 
00523 # if it's last iteration, apply chamber motion policy
00524 if [ \"$ALIGNMENT_ITERATION\" == \"%(ITERATIONS)s\" ]; then
00525   # convert this iteration's geometry into detailed xml
00526   ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s.db %(directory)s%(director)s_extra.xml --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
00527   # perform motion policy 
00528   ./Alignment/MuonAlignmentAlgorithms/scripts/motionPolicyChamber.py \
00529       %(INITIALXML)s  %(directory)s%(director)s_extra.xml \
00530       %(directory)s%(director)s_report.py \
00531       %(directory)s%(director)s_final.xml \
00532       --nsigma %(theNSigma)s
00533   # convert the resulting xml into the final sqlite geometry
00534   ./Alignment/MuonAlignmentAlgorithms/scripts/convertSQLiteXML.py %(directory)s%(director)s_final.xml %(directory)s%(director)s_final.db --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
00535 fi
00536 
00537 """ % my_vars)
00538 
00539 #####################################################################
00540 
00541 def writeValidationCfg(fname, my_vars):
00542     file(fname, "w").write("""#!/bin/sh
00543 # %(commandline)s
00544 
00545 export ALIGNMENT_CAFDIR=`pwd`
00546 mkdir files
00547 mkdir out
00548 
00549 cd %(pwd)s
00550 eval `scramv1 run -sh`
00551 ALIGNMENT_AFSDIR=`pwd`
00552 ALIGNMENT_ITERATION=%(iteration)d
00553 ALIGNMENT_MAPPLOTS=None
00554 ALIGNMENT_SEGDIFFPLOTS=None
00555 ALIGNMENT_CURVATUREPLOTS=None
00556 ALIGNMENT_EXTRAPLOTS=%(extraPlots)s
00557 export ALIGNMENT_GPRCDCONNECT=%(gprcdconnect)s
00558 export ALIGNMENT_GPRCD=%(gprcd)s
00559 export ALIGNMENT_DO_DT=%(doDT)s
00560 export ALIGNMENT_DO_CSC=%(doCSC)s
00561 
00562 
00563 # copy the scripts to CAFDIR
00564 cd Alignment/MuonAlignmentAlgorithms/scripts/
00565 cp -f plotscripts.py $ALIGNMENT_CAFDIR/
00566 cp -f mutypes.py $ALIGNMENT_CAFDIR/
00567 cp -f alignmentValidation.py $ALIGNMENT_CAFDIR/
00568 cp -f phiedges_fitfunctions.C $ALIGNMENT_CAFDIR/
00569 cp -f createTree.py $ALIGNMENT_CAFDIR/
00570 cp -f signConventions.py $ALIGNMENT_CAFDIR/
00571 cp -f convertSQLiteXML.py $ALIGNMENT_CAFDIR/
00572 cp -f wrapperExtraPlots.sh $ALIGNMENT_CAFDIR/
00573 cd -
00574 cp Alignment/MuonAlignmentAlgorithms/test/browser/tree* $ALIGNMENT_CAFDIR/out/
00575 
00576 # copy the results to CAFDIR
00577 cp -f %(directory1)s%(director1)s_report.py $ALIGNMENT_CAFDIR/files/
00578 cp -f %(directory)s%(director)s_report.py $ALIGNMENT_CAFDIR/files/
00579 cp -f %(directory1)s%(director1)s.root $ALIGNMENT_CAFDIR/files/
00580 cp -f %(directory)s%(director)s.root $ALIGNMENT_CAFDIR/files/
00581 if [ -e %(directory1)s%(director1)s_plotting.root ] && [ -e %(directory)s%(director)s_plotting.root ]; then
00582   cp -f %(directory1)s%(director1)s_plotting.root $ALIGNMENT_CAFDIR/files/
00583   cp -f %(directory)s%(director)s_plotting.root $ALIGNMENT_CAFDIR/files/
00584   ALIGNMENT_MAPPLOTS=%(mapplots)s
00585   ALIGNMENT_SEGDIFFPLOTS=%(segdiffplots)s
00586   ALIGNMENT_CURVATUREPLOTS=%(curvatureplots)s
00587 fi
00588 
00589 dtcsc=""
00590 if [ $ALIGNMENT_DO_DT == \"True\" ]; then
00591   dtcsc="--dt"
00592 fi
00593 if [ $ALIGNMENT_DO_CSC == \"True\" ]; then
00594   dtcsc="${dtcsc} --csc"
00595 fi
00596 
00597 
00598 cd $ALIGNMENT_CAFDIR/
00599 echo \" ### Start running ###\"
00600 date
00601 
00602 # do fits and median plots first 
00603 ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out  --createDirSructure --dt --csc --fit --median
00604 
00605 if [ $ALIGNMENT_MAPPLOTS == \"True\" ]; then
00606   ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out  $dtcsc --map
00607 fi
00608 
00609 if [ $ALIGNMENT_SEGDIFFPLOTS == \"True\" ]; then
00610   ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out  $dtcsc --segdiff
00611 fi                   
00612 
00613 if [ $ALIGNMENT_CURVATUREPLOTS == \"True\" ]; then
00614   ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out  $dtcsc --curvature
00615 fi
00616 
00617 if [ $ALIGNMENT_EXTRAPLOTS == \"True\" ]; then
00618   if [ \"zzz%(copytrackerdb)s\" != \"zzz\" ]; then
00619     cp -f $ALIGNMENT_AFSDIR/%(copytrackerdb)s $ALIGNMENT_CAFDIR/
00620   fi
00621   cp $ALIGNMENT_AFSDIR/inertGlobalPositionRcd.db .
00622   ./convertSQLiteXML.py $ALIGNMENT_AFSDIR/%(INITIALGEOM)s g0.xml --noLayers  --gprcdconnect $ALIGNMENT_GPRCDCONNECT --gprcd $ALIGNMENT_GPRCD
00623   ./wrapperExtraPlots.sh -n $ALIGNMENT_ITERATION -i $ALIGNMENT_AFSDIR -0 g0.xml -z -w %(station123params)s %(dir_no_)s
00624   mkdir out/extra
00625   cd %(dir_no_)s
00626   mv MB ../out/extra/
00627   mv ME ../out/extra/
00628   cd -
00629 fi
00630 
00631 # run simple diagnostic
00632 ./alignmentValidation.py -l %(validationLabel)s -i $ALIGNMENT_CAFDIR --i1 files --iN files --i1prefix %(director1)s --iNprefix %(director)s -o $ALIGNMENT_CAFDIR/out --dt --csc --diagnostic
00633 
00634 # fill the tree browser structure: 
00635 ./createTree.py -i $ALIGNMENT_CAFDIR/out
00636 
00637 timestamp=`date \"+%%y-%%m-%%d %%H:%%M:%%S\"`
00638 echo \"%(validationLabel)s.plots (${timestamp})\" > out/label.txt
00639 
00640 ls -l out/
00641 timestamp=`date +%%Y%%m%%d%%H%%M%%S`
00642 tar czf %(validationLabel)s_${timestamp}.tgz out
00643 cp -f %(validationLabel)s_${timestamp}.tgz $ALIGNMENT_AFSDIR/
00644 
00645 """ % my_vars)
00646 
00647 
00648 #####################################################################
00649 
00650 #SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS = True
00651 SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS = False
00652 
00653 bsubfile = ["#!/bin/sh", ""]
00654 bsubnames = []
00655 last_align = None
00656 directory = ""
00657 
00658 for iteration in range(1, ITERATIONS+1):
00659     if iteration == 1:
00660         inputdb = INITIALGEOM
00661         inputdbdir = directory[:]
00662     else:
00663         inputdb = director + ".db"
00664         inputdbdir = directory[:]
00665 
00666     directory = "%s%02d/" % (DIRNAME, iteration)
00667     director = directory[:-1]
00668 
00669     dir_no_ = DIRNAME
00670     if DIRNAME[-1]=='_': dir_no_ = DIRNAME[:-1]
00671  
00672     os.system("rm -rf %s; mkdir %s" % (directory, directory))
00673     os.system("cp Alignment/MuonAlignmentAlgorithms/python/gather_cfg.py %s" % directory)
00674     os.system("cp Alignment/MuonAlignmentAlgorithms/python/align_cfg.py %s" % directory)
00675 
00676     bsubfile.append("cd %s" % directory)
00677 
00678     mapplots = False
00679     if mapplots_ingeneral and (iteration == 1 or iteration == 3 or iteration == 5 or iteration == 7 or iteration == 9 or iteration == ITERATIONS): mapplots = True
00680     segdiffplots = False
00681     if segdiffplots_ingeneral and (iteration == 1 or iteration == ITERATIONS): segdiffplots = True
00682     curvatureplots = False
00683     if curvatureplots_ingeneral and (iteration == 1 or iteration == ITERATIONS): curvatureplots = True
00684 
00685     ### gather.sh runners for njobs
00686     for jobnumber in range(njobs):
00687         if not options.inputInBlocks:
00688           inputfiles = " ".join(fileNames[jobnumber*stepsize:(jobnumber+1)*stepsize])
00689         else:
00690           inputfiles = " ".join(fileNamesBlocks[jobnumber])
00691         
00692         if mapplots or segdiffplots or curvatureplots: copyplots = "plotting*.root"
00693         else: copyplots = ""
00694 
00695         if len(inputfiles) > 0:
00696             gather_fileName = "%sgather%03d.sh" % (directory, jobnumber)
00697             writeGatherCfg(gather_fileName, vars())
00698             os.system("chmod +x %s" % gather_fileName)
00699             bsubfile.append("echo %sgather%03d.sh" % (directory, jobnumber))
00700 
00701             if last_align is None: waiter = ""
00702             else: waiter = "-w \"ended(%s)\"" % last_align            
00703             if options.big: queue = "cmscaf1nd"
00704             else: queue = "cmscaf1nh"
00705 
00706             if user_mail: bsubfile.append("bsub -R \"type==SLC5_64\" -q %s -J \"%s_gather%03d\" -u %s %s gather%03d.sh" % (queue, director, jobnumber, user_mail, waiter, jobnumber))
00707             else: bsubfile.append("bsub -R \"type==SLC5_64\" -q %s -J \"%s_gather%03d\" %s gather%03d.sh" % (queue, director, jobnumber, waiter, jobnumber))
00708 
00709             bsubnames.append("ended(%s_gather%03d)" % (director, jobnumber))
00710 
00711 
00712     ### align.sh
00713     if SUPER_SPECIAL_XY_AND_DXDZ_ITERATIONS:
00714         if ( iteration == 1 or iteration == 3 or iteration == 5 or iteration == 7 or iteration == 9):
00715             tmp = station123params, station123params, useResiduals 
00716             station123params, station123params, useResiduals = "000010", "000010", "0010"
00717             writeAlignCfg("%salign.sh" % directory, vars())
00718             station123params, station123params, useResiduals = tmp
00719         elif ( iteration == 2 or iteration == 4 or iteration == 6 or iteration == 8 or iteration == 10):
00720             tmp = station123params, station123params, useResiduals 
00721             station123params, station123params, useResiduals = "110001", "100001", "1100"
00722             writeAlignCfg("%salign.sh" % directory, vars())
00723             station123params, station123params, useResiduals = tmp
00724     else:
00725         writeAlignCfg("%salign.sh" % directory, vars())
00726     
00727     os.system("chmod +x %salign.sh" % directory)
00728 
00729     bsubfile.append("echo %salign.sh" % directory)
00730     if user_mail: bsubfile.append("bsub -R \"type==SLC5_64\" -q cmscaf1nd -J \"%s_align\" -u %s -w \"%s\" align.sh" % (director, user_mail, " && ".join(bsubnames)))
00731     else: bsubfile.append("bsub -R \"type==SLC5_64\" -q cmscaf1nd -J \"%s_align\" -w \"%s\" align.sh" % (director, " && ".join(bsubnames)))
00732     
00733     #bsubfile.append("cd ..")
00734     bsubnames = []
00735     last_align = "%s_align" % director
00736     
00737     
00738     ### after the last iteration (optionally) do diagnostics run
00739     if len(validationLabel) and iteration == ITERATIONS:
00740         # do we have plotting files created?
00741         directory1 = "%s01/" % DIRNAME
00742         director1 = directory1[:-1]
00743 
00744         writeValidationCfg("%svalidation.sh" % directory, vars())
00745         os.system("chmod +x %svalidation.sh" % directory)
00746         
00747         bsubfile.append("echo %svalidation.sh" % directory)
00748         if user_mail: bsubfile.append("bsub -R \"type==SLC5_64\" -q cmscaf1nd -J \"%s_validation\" -u %s -w \"ended(%s)\" validation.sh" % (director, user_mail, last_align))
00749         else: bsubfile.append("bsub -R \"type==SLC5_64\" -q cmscaf1nd -J \"%s_validation\" -w \"ended(%s)\" validation.sh" % (director, last_align))
00750 
00751     bsubfile.append("cd ..")
00752     bsubfile.append("")
00753 
00754 
00755 file(options.submitJobs, "w").write("\n".join(bsubfile))
00756 os.system("chmod +x %s" % options.submitJobs)