|
|
|
Public Member Functions | |
| def | __init__ |
| def | createWorkFlows |
| def | makeCmd |
| def | prepare |
| def | readMatrix |
| def | reset |
| def | show |
| def | showRaw |
| def | showWorkFlows |
| def | updateDB |
Public Attributes | |
| addCommand | |
| apply | |
| commandLineWf | |
| files | |
| filesPrefMap | |
| nameList | |
| noRun | |
| overWrite | |
| relvalModule | |
| what | |
| wm | |
| workFlows | |
| workFlowSteps | |
Definition at line 16 of file MatrixReader.py.
| def MatrixReader::MatrixReader::__init__ | ( | self, | |
| opt | |||
| ) |
Definition at line 18 of file MatrixReader.py.
| def MatrixReader::MatrixReader::createWorkFlows | ( | self, | |
| fileNameIn | |||
| ) |
Definition at line 395 of file MatrixReader.py.
00396 : 00397 00398 prefixIn = self.filesPrefMap[fileNameIn] 00399 00400 # get through the list of items and update the requested workflows only 00401 keyList = self.workFlowSteps.keys() 00402 ids = [] 00403 for item in keyList: 00404 id, pref = item 00405 if pref != prefixIn : continue 00406 ids.append(id) 00407 ids.sort() 00408 for key in ids: 00409 val = self.workFlowSteps[(key,prefixIn)] 00410 num, name, commands, stepList = val 00411 nameId = str(num)+'_'+name 00412 if nameId in self.nameList: 00413 print "==> duplicate name found for ", nameId 00414 print ' keeping : ', self.nameList[nameId] 00415 print ' ignoring : ', val 00416 else: 00417 self.nameList[nameId] = val 00418 00419 self.workFlows.append(WorkFlow(num, name, commands=commands)) 00420 00421 return
| def MatrixReader::MatrixReader::makeCmd | ( | self, | |
| step | |||
| ) |
Definition at line 65 of file MatrixReader.py.
00066 : 00067 00068 cmd = '' 00069 cfg = None 00070 input = None 00071 for k,v in step.items(): 00072 if 'no_exec' in k : continue # we want to really run it ... 00073 if k.lower() == 'cfg': 00074 cfg = v 00075 continue # do not append to cmd, return separately 00076 if k.lower() == 'input': 00077 input = v 00078 continue # do not append to cmd, return separately 00079 00080 #chain the configs 00081 #if k.lower() == '--python': 00082 # v = 'step%d_%s'%(index,v) 00083 cmd += ' ' + k + ' ' + str(v) 00084 return cfg, input, cmd
| def MatrixReader::MatrixReader::prepare | ( | self, | |
useInput = None, |
|||
refRel = '', |
|||
fromScratch = None |
|||
| ) |
Definition at line 422 of file MatrixReader.py.
00423 : 00424 00425 for matrixFile in self.files: 00426 if self.what != 'all' and self.what not in matrixFile: 00427 print "ignoring non-requested file",matrixFile 00428 continue 00429 if self.what == 'all' and ('upgrade' in matrixFile): 00430 print "ignoring",matrixFile,"from default matrix" 00431 continue 00432 00433 try: 00434 self.readMatrix(matrixFile, useInput, refRel, fromScratch) 00435 except Exception, e: 00436 print "ERROR reading file:", matrixFile, str(e) 00437 raise 00438 00439 try: 00440 self.createWorkFlows(matrixFile) 00441 except Exception, e: 00442 print "ERROR creating workflows :", str(e) 00443 raise 00444
| def MatrixReader::MatrixReader::readMatrix | ( | self, | |
| fileNameIn, | |||
useInput = None, |
|||
refRel = None, |
|||
fromScratch = None |
|||
| ) |
Definition at line 85 of file MatrixReader.py.
00086 : 00087 00088 prefix = self.filesPrefMap[fileNameIn] 00089 00090 print "processing ", fileNameIn 00091 00092 try: 00093 _tmpMod = __import__( 'Configuration.PyReleaseValidation.'+fileNameIn ) 00094 self.relvalModule = sys.modules['Configuration.PyReleaseValidation.'+fileNameIn] 00095 except Exception, e: 00096 print "ERROR importing file ", fileNameIn, str(e) 00097 return 00098 00099 print "request for INPUT for ", useInput 00100 00101 00102 fromInput={} 00103 00104 if useInput: 00105 for i in useInput: 00106 if ':' in i: 00107 (ik,il)=i.split(':') 00108 if ik=='all': 00109 for k in self.relvalModule.workflows.keys(): 00110 fromInput[float(k)]=int(il) 00111 else: 00112 fromInput[float(ik)]=int(il) 00113 else: 00114 if i=='all': 00115 for k in self.relvalModule.workflows.keys(): 00116 fromInput[float(k)]=0 00117 else: 00118 fromInput[float(i)]=0 00119 00120 if fromScratch: 00121 fromScratch=map(float,fromScratch) 00122 for num in fromScratch: 00123 if num in fromInput: 00124 fromInput.pop(num) 00125 #overwrite steps 00126 if self.overWrite: 00127 for p in self.overWrite: 00128 self.relvalModule.steps.overwrite(p) 00129 00130 #change the origin of dataset on the fly 00131 if refRel: 00132 if ',' in refRel: 00133 refRels=refRel.split(',') 00134 if len(refRels)!=len(self.relvalModule.baseDataSetRelease): 00135 return 00136 self.relvalModule.changeRefRelease( 00137 self.relvalModule.steps, 00138 zip(self.relvalModule.baseDataSetRelease,refRels) 00139 ) 00140 else: 00141 self.relvalModule.changeRefRelease( 00142 self.relvalModule.steps, 00143 [(x,refRel) for x in self.relvalModule.baseDataSetRelease] 00144 ) 00145 00146 00147 for num, wfInfo in self.relvalModule.workflows.items(): 00148 commands=[] 00149 wfName = wfInfo[0] 00150 stepList = wfInfo[1] 00151 # if no explicit name given for the workflow, use the name of step1 00152 if wfName.strip() == '': wfName = stepList[0] 00153 # option to specialize the wf as the third item in the WF list 00154 addTo=None 00155 addCom=None 00156 if len(wfInfo)>=3: 00157 addCom=wfInfo[2] 00158 if not type(addCom)==list: addCom=[addCom] 00159 #print 'added dict',addCom 00160 if len(wfInfo)>=4: 00161 addTo=wfInfo[3] 00162 #pad with 0 00163 while len(addTo)!=len(stepList): 00164 addTo.append(0) 00165 00166 name=wfName 00167 stepIndex=0 00168 ranStepList=[] 00169 00170 #first resolve INPUT possibilities 00171 if num in fromInput: 00172 ilevel=fromInput[num] 00173 #print num,ilevel 00174 for (stepIr,step) in enumerate(reversed(stepList)): 00175 stepName=step 00176 stepI=(len(stepList)-stepIr)-1 00177 #print stepIr,step,stepI,ilevel 00178 if stepI>ilevel: 00179 #print "ignoring" 00180 continue 00181 if stepI!=0: 00182 testName='__'.join(stepList[0:stepI+1])+'INPUT' 00183 else: 00184 testName=step+'INPUT' 00185 #print "JR",stepI,stepIr,testName,stepList 00186 if testName in self.relvalModule.steps.keys(): 00187 #print "JR",stepI,stepIr 00188 stepList[stepI]=testName 00189 #pop the rest in the list 00190 #print "\tmod prepop",stepList 00191 for p in range(stepI): 00192 stepList.pop(0) 00193 #print "\t\tmod",stepList 00194 break 00195 00196 00197 for (stepI,step) in enumerate(stepList): 00198 stepName=step 00199 if self.wm: 00200 #cannot put a certain number of things in wm 00201 if stepName in [ 00202 #'HARVEST','HARVESTD','HARVESTDreHLT', 00203 'RECODFROMRAWRECO','SKIMD','SKIMCOSD','SKIMDreHLT' 00204 ]: 00205 continue 00206 00207 #replace stepName is needed 00208 #if stepName in self.replaceStep 00209 if len(name) > 0 : name += '+' 00210 #any step can be mirrored with INPUT 00211 ## maybe we want too level deep input 00212 """ 00213 if num in fromInput: 00214 if step+'INPUT' in self.relvalModule.steps.keys(): 00215 stepName = step+"INPUT" 00216 stepList.remove(step) 00217 stepList.insert(stepIndex,stepName) 00218 """ 00219 name += stepName 00220 00221 if addCom and (not addTo or addTo[stepIndex]==1): 00222 from Configuration.PyReleaseValidation.relval_steps import merge 00223 copyStep=merge(addCom+[self.relvalModule.steps[stepName]]) 00224 cfg, input, opts = self.makeCmd(copyStep) 00225 else: 00226 cfg, input, opts = self.makeCmd(self.relvalModule.steps[stepName]) 00227 00228 if input and cfg : 00229 msg = "FATAL ERROR: found both cfg and input for workflow "+str(num)+' step '+stepName 00230 raise MatrixException(msg) 00231 00232 if input: 00233 cmd = input 00234 if self.noRun: 00235 cmd.run=[] 00236 else: 00237 if cfg: 00238 cmd = 'cmsDriver.py '+cfg+' '+opts 00239 else: 00240 cmd = 'cmsDriver.py step'+str(stepIndex+1)+' '+opts 00241 if self.wm: 00242 cmd+=' --io %s.io --python %s.py'%(stepName,stepName) 00243 if self.addCommand: 00244 if self.apply: 00245 if stepIndex in self.apply or stepName in self.apply: 00246 cmd +=' '+self.addCommand 00247 else: 00248 cmd +=' '+self.addCommand 00249 if self.wm: 00250 cmd=cmd.replace('DQMROOT','DQM') 00251 cmd=cmd.replace('--filetype DQM','') 00252 commands.append(cmd) 00253 ranStepList.append(stepName) 00254 stepIndex+=1 00255 00256 self.workFlowSteps[(num,prefix)] = (num, name, commands, ranStepList) 00257 00258 return 00259
| def MatrixReader::MatrixReader::reset | ( | self, | |
what = 'all' |
|||
| ) |
Definition at line 31 of file MatrixReader.py.
00032 : 00033 00034 self.what = what 00035 00036 #a bunch of information, but not yet the WorkFlow object 00037 self.workFlowSteps = {} 00038 #the actual WorkFlow objects 00039 self.workFlows = [] 00040 self.nameList = {} 00041 00042 self.filesPrefMap = {'relval_standard' : 'std-' , 00043 'relval_highstats': 'hi-' , 00044 'relval_pileup': 'PU-' , 00045 'relval_generator': 'gen-' , 00046 'relval_production': 'prod-' , 00047 'relval_ged': 'ged-', 00048 'relval_upgrade':'upg-', 00049 'relval_identity':'id-' 00050 } 00051 00052 self.files = ['relval_standard' , 00053 'relval_highstats', 00054 'relval_pileup', 00055 'relval_generator', 00056 'relval_production', 00057 'relval_ged', 00058 'relval_upgrade', 00059 'relval_identity' 00060 ] 00061 00062 self.relvalModule = None 00063 00064 return
| def MatrixReader::MatrixReader::show | ( | self, | |
selected = None, |
|||
extended = True |
|||
| ) |
Definition at line 445 of file MatrixReader.py.
00445 : 00446 00447 self.showWorkFlows(selected,extended) 00448 print '\n','-'*80,'\n' 00449 00450
| def MatrixReader::MatrixReader::showRaw | ( | self, | |
| useInput, | |||
refRel = None, |
|||
fromScratch = None, |
|||
what = 'all', |
|||
step1Only = False, |
|||
selected = None |
|||
| ) |
Definition at line 260 of file MatrixReader.py.
00261 : 00262 00263 if selected: 00264 selected=map(float,selected) 00265 for matrixFile in self.files: 00266 00267 self.reset(what) 00268 00269 if self.what != 'all' and self.what not in matrixFile: 00270 print "ignoring non-requested file",matrixFile 00271 continue 00272 00273 try: 00274 self.readMatrix(matrixFile, useInput, refRel, fromScratch) 00275 except Exception, e: 00276 print "ERROR reading file:", matrixFile, str(e) 00277 raise 00278 00279 if not self.workFlowSteps: continue 00280 00281 dataFileName = matrixFile.replace('relval_', 'cmsDriver_')+'_hlt.txt' 00282 outFile = open(dataFileName,'w') 00283 00284 print "found ", len(self.workFlowSteps.keys()), ' workflows for ', dataFileName 00285 ids = self.workFlowSteps.keys() 00286 ids.sort() 00287 indexAndSteps=[] 00288 00289 writtenWF=0 00290 for key in ids: 00291 if selected and not (key[0] in selected): 00292 continue 00293 #trick to skip the HImix IB test 00294 if key[0]==203.1 or key[0]==204.1 or key[0]==205.1 or key[0]==4.51 or key[0]==4.52: continue 00295 num, name, commands, stepList = self.workFlowSteps[key] 00296 00297 wfName,stepNames= name.split('+',1) 00298 00299 stepNames=stepNames.replace('+RECODFROMRAWRECO','') 00300 stepNames=stepNames.replace('+SKIMCOSD','') 00301 stepNames=stepNames.replace('+SKIMD','') 00302 if 'HARVEST' in stepNames: 00303 #find out automatically what to remove 00304 exactb=stepNames.index('+HARVEST') 00305 exacte=stepNames.index('+',exactb+1) if ('+' in stepNames[exactb+1:]) else (len(stepNames)) 00306 stepNames=stepNames.replace(stepNames[exactb:exacte],'') 00307 otherSteps = None 00308 if '+' in stepNames: 00309 step1,otherSteps = stepNames.split('+',1) 00310 00311 line = str(num) + ' ++ '+ wfName 00312 if otherSteps and not step1Only: 00313 line += ' ++ ' +otherSteps.replace('+',',') 00314 else: 00315 line += ' ++ none' 00316 inputInfo=None 00317 if not isinstance(commands[0],str): 00318 inputInfo=commands[0] 00319 if otherSteps: 00320 for (i,c) in enumerate(otherSteps.split('+')): 00321 #pad with set 00322 for p in range(len(indexAndSteps),i+2): 00323 indexAndSteps.append(set()) 00324 indexAndSteps[i+1].add((c,commands[i+1])) 00325 00326 if inputInfo : 00327 #skip the samples from INPUT when step1Only is on 00328 if step1Only: continue 00329 line += ' ++ REALDATA: '+inputInfo.dataSet 00330 if inputInfo.run!=[]: line += ', RUN:'+'|'.join(map(str,inputInfo.run)) 00331 line += ', FILES: ' +str(inputInfo.files) 00332 line += ', EVENTS: '+str(inputInfo.events) 00333 if inputInfo.label!='': 00334 line += ', LABEL: ' +inputInfo.label 00335 line += ', LOCATION:'+inputInfo.location 00336 line += ' @@@' 00337 else: 00338 line += ' @@@ '+commands[0] 00339 line=line.replace('DQMROOT','DQM') 00340 writtenWF+=1 00341 outFile.write(line+'\n') 00342 00343 00344 outFile.write('\n'+'\n') 00345 if step1Only: continue 00346 00347 for (index,s) in enumerate(indexAndSteps): 00348 for (stepName,cmd) in s: 00349 stepIndex=index+1 00350 if 'dbsquery.log' in cmd: continue 00351 line = 'STEP%d ++ '%(stepIndex,) +stepName + ' @@@ '+cmd 00352 line=line.replace('DQMROOT','DQM') 00353 outFile.write(line+'\n') 00354 outFile.write('\n'+'\n') 00355 outFile.close() 00356 print "wrote ",writtenWF, ' workflow'+('s' if (writtenWF!=1) else ''),' to ', outFile.name 00357 return 00358
| def MatrixReader::MatrixReader::showWorkFlows | ( | self, | |
selected = None, |
|||
extended = True |
|||
| ) |
Definition at line 359 of file MatrixReader.py.
00360 : 00361 if selected: selected = map(float,selected) 00362 maxLen = 100 # for summary, limit width of output 00363 fmt1 = "%-6s %-35s [1]: %s ..." 00364 fmt2 = " %35s [%d]: %s ..." 00365 print "\nfound a total of ", len(self.workFlows), ' workflows:' 00366 if selected: 00367 print " of which the following", len(selected), 'were selected:' 00368 #-ap for now: 00369 maxLen = -1 # for individual listing, no limit on width 00370 fmt1 = "%-6s %-35s [1]: %s " 00371 fmt2 = " %35s [%d]: %s" 00372 00373 N=[] 00374 for wf in self.workFlows: 00375 if selected and float(wf.numId) not in selected: continue 00376 if extended: print '' 00377 #pad with zeros 00378 for i in range(len(N),len(wf.cmds)): N.append(0) 00379 N[len(wf.cmds)-1]+=1 00380 wfName, stepNames = wf.nameId.split('+',1) 00381 for i,s in enumerate(wf.cmds): 00382 if extended: 00383 if i==0: 00384 print fmt1 % (wf.numId, stepNames, (str(s)+' ')[:maxLen]) 00385 else: 00386 print fmt2 % ( ' ', i+1, (str(s)+' ')[:maxLen]) 00387 else: 00388 print "%-6s %-35s "% (wf.numId, stepNames) 00389 break 00390 print '' 00391 for i,n in enumerate(N): 00392 if n: print n,'workflows with',i+1,'steps' 00393 00394 return
| def MatrixReader::MatrixReader::updateDB | ( | self | ) |
Definition at line 451 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 31 of file MatrixReader.py.
Definition at line 31 of file MatrixReader.py.
Definition at line 31 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 31 of file MatrixReader.py.
Definition at line 31 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 31 of file MatrixReader.py.
Definition at line 31 of file MatrixReader.py.
1.7.3