|
|
|
Public Member Functions | |
| def | __init__ |
| def | createWorkFlows |
| def | makeCmd |
| def | prepare |
| def | readMatrix |
| def | reset |
| def | show |
| def | showRaw |
| def | showWorkFlows |
| def | updateDB |
Public Attributes | |
| addCommand | |
| commandLineWf | |
| files | |
| filesPrefMap | |
| nameList | |
| overWrite | |
| relvalModule | |
| what | |
| wm | |
| workFlows | |
| workFlowSteps | |
Definition at line 16 of file MatrixReader.py.
| def MatrixReader::MatrixReader::__init__ | ( | self, | |
| opt | |||
| ) |
Definition at line 18 of file MatrixReader.py.
| def MatrixReader::MatrixReader::createWorkFlows | ( | self, | |
| fileNameIn | |||
| ) |
Definition at line 381 of file MatrixReader.py.
00382 : 00383 00384 prefixIn = self.filesPrefMap[fileNameIn] 00385 00386 # get through the list of items and update the requested workflows only 00387 keyList = self.workFlowSteps.keys() 00388 ids = [] 00389 for item in keyList: 00390 id, pref = item 00391 if pref != prefixIn : continue 00392 ids.append(id) 00393 ids.sort() 00394 for key in ids: 00395 val = self.workFlowSteps[(key,prefixIn)] 00396 num, name, commands, stepList = val 00397 nameId = str(num)+'_'+name 00398 if nameId in self.nameList: 00399 print "==> duplicate name found for ", nameId 00400 print ' keeping : ', self.nameList[nameId] 00401 print ' ignoring : ', val 00402 else: 00403 self.nameList[nameId] = val 00404 00405 self.workFlows.append(WorkFlow(num, name, commands=commands)) 00406 00407 return
| def MatrixReader::MatrixReader::makeCmd | ( | self, | |
| step | |||
| ) |
Definition at line 61 of file MatrixReader.py.
00062 : 00063 00064 cmd = '' 00065 cfg = None 00066 input = None 00067 for k,v in step.items(): 00068 if 'no_exec' in k : continue # we want to really run it ... 00069 if k.lower() == 'cfg': 00070 cfg = v 00071 continue # do not append to cmd, return separately 00072 if k.lower() == 'input': 00073 input = v 00074 continue # do not append to cmd, return separately 00075 00076 #chain the configs 00077 #if k.lower() == '--python': 00078 # v = 'step%d_%s'%(index,v) 00079 cmd += ' ' + k + ' ' + str(v) 00080 return cfg, input, cmd
| def MatrixReader::MatrixReader::prepare | ( | self, | |
useInput = None, |
|||
refRel = '', |
|||
fromScratch = None |
|||
| ) |
Definition at line 408 of file MatrixReader.py.
00409 : 00410 00411 for matrixFile in self.files: 00412 if self.what != 'all' and self.what not in matrixFile: 00413 print "ignoring non-requested file",matrixFile 00414 continue 00415 00416 try: 00417 self.readMatrix(matrixFile, useInput, refRel, fromScratch) 00418 except Exception, e: 00419 print "ERROR reading file:", matrixFile, str(e) 00420 raise 00421 00422 try: 00423 self.createWorkFlows(matrixFile) 00424 except Exception, e: 00425 print "ERROR creating workflows :", str(e) 00426 raise 00427
| def MatrixReader::MatrixReader::readMatrix | ( | self, | |
| fileNameIn, | |||
useInput = None, |
|||
refRel = None, |
|||
fromScratch = None |
|||
| ) |
Definition at line 81 of file MatrixReader.py.
00082 : 00083 00084 prefix = self.filesPrefMap[fileNameIn] 00085 00086 print "processing ", fileNameIn 00087 00088 try: 00089 _tmpMod = __import__( 'Configuration.PyReleaseValidation.'+fileNameIn ) 00090 self.relvalModule = sys.modules['Configuration.PyReleaseValidation.'+fileNameIn] 00091 except Exception, e: 00092 print "ERROR importing file ", fileNameIn, str(e) 00093 return 00094 00095 print "request for INPUT for ", useInput 00096 00097 00098 fromInput={} 00099 00100 if useInput: 00101 for i in useInput: 00102 if ':' in i: 00103 (ik,il)=i.split(':') 00104 if ik=='all': 00105 for k in self.relvalModule.workflows.keys(): 00106 fromInput[float(k)]=int(il) 00107 else: 00108 fromInput[float(ik)]=int(il) 00109 else: 00110 if i=='all': 00111 for k in self.relvalModule.workflows.keys(): 00112 fromInput[float(k)]=0 00113 else: 00114 fromInput[float(i)]=0 00115 00116 if fromScratch: 00117 fromScratch=map(float,fromScratch) 00118 for num in fromScratch: 00119 if num in fromInput: 00120 fromInput.pop(num) 00121 #overwrite steps 00122 if self.overWrite: 00123 for p in self.overWrite: 00124 self.relvalModule.steps.overwrite(p) 00125 00126 #change the origin of dataset on the fly 00127 if refRel: 00128 if ',' in refRel: 00129 refRels=refRel.split(',') 00130 if len(refRels)!=len(self.relvalModule.baseDataSetRelease): 00131 return 00132 self.relvalModule.changeRefRelease( 00133 self.relvalModule.steps, 00134 zip(self.relvalModule.baseDataSetRelease,refRels) 00135 ) 00136 else: 00137 self.relvalModule.changeRefRelease( 00138 self.relvalModule.steps, 00139 [(x,refRel) for x in self.relvalModule.baseDataSetRelease] 00140 ) 00141 00142 00143 for num, wfInfo in self.relvalModule.workflows.items(): 00144 commands=[] 00145 wfName = wfInfo[0] 00146 stepList = wfInfo[1] 00147 # if no explicit name given for the workflow, use the name of step1 00148 if wfName.strip() == '': wfName = stepList[0] 00149 # option to specialize the wf as the third item in the WF list 00150 addTo=None 00151 addCom=None 00152 if len(wfInfo)>=3: 00153 addCom=wfInfo[2] 00154 if not type(addCom)==list: addCom=[addCom] 00155 #print 'added dict',addCom 00156 if len(wfInfo)>=4: 00157 addTo=wfInfo[3] 00158 #pad with 0 00159 while len(addTo)!=len(stepList): 00160 addTo.append(0) 00161 00162 name=wfName 00163 stepIndex=0 00164 ranStepList=[] 00165 00166 #first resolve INPUT possibilities 00167 if num in fromInput: 00168 ilevel=fromInput[num] 00169 #print num,ilevel 00170 for (stepIr,step) in enumerate(reversed(stepList)): 00171 stepName=step 00172 stepI=(len(stepList)-stepIr)-1 00173 #print stepIr,step,stepI,ilevel 00174 if stepI>ilevel: 00175 #print "ignoring" 00176 continue 00177 if stepI!=0: 00178 testName='__'.join(stepList[0:stepI+1])+'INPUT' 00179 else: 00180 testName=step+'INPUT' 00181 #print "JR",stepI,stepIr,testName,stepList 00182 if testName in self.relvalModule.steps.keys(): 00183 #print "JR",stepI,stepIr 00184 stepList[stepI]=testName 00185 #pop the rest in the list 00186 #print "\tmod prepop",stepList 00187 for p in range(stepI): 00188 stepList.pop(0) 00189 #print "\t\tmod",stepList 00190 break 00191 00192 00193 for (stepI,step) in enumerate(stepList): 00194 stepName=step 00195 if self.wm: 00196 #cannot put a certain number of things in wm 00197 if stepName in ['HARVEST','HARVESTD','HARVESTDreHLT','RECODFROMRAWRECO','SKIMD','SKIMCOSD','SKIMDreHLT']: 00198 continue 00199 00200 #replace stepName is needed 00201 #if stepName in self.replaceStep 00202 if len(name) > 0 : name += '+' 00203 #any step can be mirrored with INPUT 00204 ## maybe we want too level deep input 00205 """ 00206 if num in fromInput: 00207 if step+'INPUT' in self.relvalModule.steps.keys(): 00208 stepName = step+"INPUT" 00209 stepList.remove(step) 00210 stepList.insert(stepIndex,stepName) 00211 """ 00212 name += stepName 00213 00214 if addCom and (not addTo or addTo[stepIndex]==1): 00215 from Configuration.PyReleaseValidation.relval_steps import merge 00216 copyStep=merge(addCom+[self.relvalModule.steps[stepName]]) 00217 cfg, input, opts = self.makeCmd(copyStep) 00218 else: 00219 cfg, input, opts = self.makeCmd(self.relvalModule.steps[stepName]) 00220 00221 if input and cfg : 00222 msg = "FATAL ERROR: found both cfg and input for workflow "+str(num)+' step '+stepName 00223 raise MatrixException(msg) 00224 00225 if input: 00226 cmd = input 00227 else: 00228 if cfg: 00229 cmd = 'cmsDriver.py '+cfg+' '+opts 00230 else: 00231 cmd = 'cmsDriver.py step'+str(stepIndex+1)+' '+opts 00232 if self.wm: 00233 cmd+=' --io %s.io --python %s.py'%(stepName,stepName) 00234 if self.addCommand: 00235 cmd +=' '+self.addCommand 00236 if self.wm: 00237 cmd=cmd.replace('DQMROOT','DQM') 00238 commands.append(cmd) 00239 ranStepList.append(stepName) 00240 stepIndex+=1 00241 00242 self.workFlowSteps[(num,prefix)] = (num, name, commands, ranStepList) 00243 00244 return 00245
| def MatrixReader::MatrixReader::reset | ( | self, | |
what = 'all' |
|||
| ) |
Definition at line 29 of file MatrixReader.py.
00030 : 00031 00032 self.what = what 00033 00034 #a bunch of information, but not yet the WorkFlow object 00035 self.workFlowSteps = {} 00036 #the actual WorkFlow objects 00037 self.workFlows = [] 00038 self.nameList = {} 00039 00040 self.filesPrefMap = {'relval_standard' : 'std-' , 00041 'relval_highstats': 'hi-' , 00042 'relval_pileup': 'PU-' , 00043 'relval_generator': 'gen-' , 00044 'relval_production': 'prod-' , 00045 'relval_ged': 'ged-', 00046 'relval_identity':'id-' 00047 } 00048 00049 self.files = ['relval_standard' , 00050 'relval_highstats', 00051 'relval_pileup', 00052 'relval_generator', 00053 'relval_production', 00054 'relval_ged', 00055 'relval_identity' 00056 ] 00057 00058 self.relvalModule = None 00059 00060 return
| def MatrixReader::MatrixReader::show | ( | self, | |
selected = None, |
|||
extended = True |
|||
| ) |
Definition at line 428 of file MatrixReader.py.
00428 : 00429 00430 self.showWorkFlows(selected,extended) 00431 print '\n','-'*80,'\n' 00432 00433
| def MatrixReader::MatrixReader::showRaw | ( | self, | |
| useInput, | |||
refRel = None, |
|||
fromScratch = None, |
|||
what = 'all', |
|||
step1Only = False, |
|||
selected = None |
|||
| ) |
Definition at line 246 of file MatrixReader.py.
00247 : 00248 00249 if selected: 00250 selected=map(float,selected) 00251 for matrixFile in self.files: 00252 00253 self.reset(what) 00254 00255 if self.what != 'all' and self.what not in matrixFile: 00256 print "ignoring non-requested file",matrixFile 00257 continue 00258 00259 try: 00260 self.readMatrix(matrixFile, useInput, refRel, fromScratch) 00261 except Exception, e: 00262 print "ERROR reading file:", matrixFile, str(e) 00263 raise 00264 00265 if not self.workFlowSteps: continue 00266 00267 dataFileName = matrixFile.replace('relval_', 'cmsDriver_')+'_hlt.txt' 00268 outFile = open(dataFileName,'w') 00269 00270 print "found ", len(self.workFlowSteps.keys()), ' workflows for ', dataFileName 00271 ids = self.workFlowSteps.keys() 00272 ids.sort() 00273 indexAndSteps=[] 00274 00275 writtenWF=0 00276 for key in ids: 00277 if selected and not (key[0] in selected): 00278 continue 00279 #trick to skip the HImix IB test 00280 if key[0]==203.1 or key[0]==204.1 or key[0]==205.1 or key[0]==4.51 or key[0]==4.52: continue 00281 num, name, commands, stepList = self.workFlowSteps[key] 00282 00283 wfName,stepNames= name.split('+',1) 00284 00285 stepNames=stepNames.replace('+RECODFROMRAWRECO','') 00286 stepNames=stepNames.replace('+SKIMCOSD','') 00287 stepNames=stepNames.replace('+SKIMD','') 00288 if 'HARVEST' in stepNames: 00289 #find out automatically what to remove 00290 exactb=stepNames.index('+HARVEST') 00291 exacte=stepNames.index('+',exactb+1) if ('+' in stepNames[exactb+1:]) else (len(stepNames)) 00292 stepNames=stepNames.replace(stepNames[exactb:exacte],'') 00293 otherSteps = None 00294 if '+' in stepNames: 00295 step1,otherSteps = stepNames.split('+',1) 00296 00297 line = str(num) + ' ++ '+ wfName 00298 if otherSteps and not step1Only: 00299 line += ' ++ ' +otherSteps.replace('+',',') 00300 else: 00301 line += ' ++ none' 00302 inputInfo=None 00303 if not isinstance(commands[0],str): 00304 inputInfo=commands[0] 00305 if otherSteps: 00306 for (i,c) in enumerate(otherSteps.split('+')): 00307 #pad with set 00308 for p in range(len(indexAndSteps),i+2): 00309 indexAndSteps.append(set()) 00310 indexAndSteps[i+1].add((c,commands[i+1])) 00311 00312 if inputInfo : 00313 #skip the samples from INPUT when step1Only is on 00314 if step1Only: continue 00315 line += ' ++ REALDATA: '+inputInfo.dataSet 00316 if inputInfo.run!=[]: line += ', RUN:'+'|'.join(map(str,inputInfo.run)) 00317 line += ', FILES: ' +str(inputInfo.files) 00318 line += ', EVENTS: '+str(inputInfo.events) 00319 if inputInfo.label!='': 00320 line += ', LABEL: ' +inputInfo.label 00321 line += ', LOCATION:'+inputInfo.location 00322 line += ' @@@' 00323 else: 00324 line += ' @@@ '+commands[0] 00325 line=line.replace('DQMROOT','DQM') 00326 writtenWF+=1 00327 outFile.write(line+'\n') 00328 00329 00330 outFile.write('\n'+'\n') 00331 if step1Only: continue 00332 00333 for (index,s) in enumerate(indexAndSteps): 00334 for (stepName,cmd) in s: 00335 stepIndex=index+1 00336 if 'dbsquery.log' in cmd: continue 00337 line = 'STEP%d ++ '%(stepIndex,) +stepName + ' @@@ '+cmd 00338 line=line.replace('DQMROOT','DQM') 00339 outFile.write(line+'\n') 00340 outFile.write('\n'+'\n') 00341 outFile.close() 00342 print "wrote ",writtenWF, ' workflow'+('s' if (writtenWF!=1) else ''),' to ', outFile.name 00343 return 00344
| def MatrixReader::MatrixReader::showWorkFlows | ( | self, | |
selected = None, |
|||
extended = True |
|||
| ) |
Definition at line 345 of file MatrixReader.py.
00346 : 00347 if selected: selected = map(float,selected) 00348 maxLen = 100 # for summary, limit width of output 00349 fmt1 = "%-6s %-35s [1]: %s ..." 00350 fmt2 = " %35s [%d]: %s ..." 00351 print "\nfound a total of ", len(self.workFlows), ' workflows:' 00352 if selected: 00353 print " of which the following", len(selected), 'were selected:' 00354 #-ap for now: 00355 maxLen = -1 # for individual listing, no limit on width 00356 fmt1 = "%-6s %-35s [1]: %s " 00357 fmt2 = " %35s [%d]: %s" 00358 00359 N=[] 00360 for wf in self.workFlows: 00361 if selected and float(wf.numId) not in selected: continue 00362 if extended: print '' 00363 #pad with zeros 00364 for i in range(len(N),len(wf.cmds)): N.append(0) 00365 N[len(wf.cmds)-1]+=1 00366 wfName, stepNames = wf.nameId.split('+',1) 00367 for i,s in enumerate(wf.cmds): 00368 if extended: 00369 if i==0: 00370 print fmt1 % (wf.numId, stepNames, (str(s)+' ')[:maxLen]) 00371 else: 00372 print fmt2 % ( ' ', i+1, (str(s)+' ')[:maxLen]) 00373 else: 00374 print "%-6s %-35s "% (wf.numId, stepNames) 00375 break 00376 print '' 00377 for i,n in enumerate(N): 00378 if n: print n,'workflows with',i+1,'steps' 00379 00380 return
| def MatrixReader::MatrixReader::updateDB | ( | self | ) |
Definition at line 434 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
1.7.3