|
|
|
Public Member Functions | |
| def | __init__ |
| def | createWorkFlows |
| def | makeCmd |
| def | prepare |
| def | readMatrix |
| def | reset |
| def | show |
| def | showRaw |
| def | showWorkFlows |
| def | updateDB |
Public Attributes | |
| addCommand | |
| commandLineWf | |
| files | |
| filesPrefMap | |
| nameList | |
| overWrite | |
| relvalModule | |
| what | |
| wm | |
| workFlows | |
| workFlowSteps | |
Definition at line 16 of file MatrixReader.py.
| def MatrixReader::MatrixReader::__init__ | ( | self, | |
| opt | |||
| ) |
Definition at line 18 of file MatrixReader.py.
| def MatrixReader::MatrixReader::createWorkFlows | ( | self, | |
| fileNameIn | |||
| ) |
Definition at line 367 of file MatrixReader.py.
00368 : 00369 00370 prefixIn = self.filesPrefMap[fileNameIn] 00371 00372 # get through the list of items and update the requested workflows only 00373 keyList = self.workFlowSteps.keys() 00374 ids = [] 00375 for item in keyList: 00376 id, pref = item 00377 if pref != prefixIn : continue 00378 ids.append(id) 00379 ids.sort() 00380 for key in ids: 00381 val = self.workFlowSteps[(key,prefixIn)] 00382 num, name, commands, stepList = val 00383 nameId = str(num)+'_'+name 00384 if nameId in self.nameList: 00385 print "==> duplicate name found for ", nameId 00386 print ' keeping : ', self.nameList[nameId] 00387 print ' ignoring : ', val 00388 else: 00389 self.nameList[nameId] = val 00390 00391 self.workFlows.append(WorkFlow(num, name, commands=commands)) 00392 00393 return
| def MatrixReader::MatrixReader::makeCmd | ( | self, | |
| step | |||
| ) |
Definition at line 59 of file MatrixReader.py.
00060 : 00061 00062 cmd = '' 00063 cfg = None 00064 input = None 00065 for k,v in step.items(): 00066 if 'no_exec' in k : continue # we want to really run it ... 00067 if k.lower() == 'cfg': 00068 cfg = v 00069 continue # do not append to cmd, return separately 00070 if k.lower() == 'input': 00071 input = v 00072 continue # do not append to cmd, return separately 00073 00074 #chain the configs 00075 #if k.lower() == '--python': 00076 # v = 'step%d_%s'%(index,v) 00077 cmd += ' ' + k + ' ' + str(v) 00078 return cfg, input, cmd
| def MatrixReader::MatrixReader::prepare | ( | self, | |
useInput = None, |
|||
refRel = '', |
|||
fromScratch = None |
|||
| ) |
Definition at line 394 of file MatrixReader.py.
00395 : 00396 00397 for matrixFile in self.files: 00398 if self.what != 'all' and self.what not in matrixFile: 00399 print "ignoring non-requested file",matrixFile 00400 continue 00401 00402 try: 00403 self.readMatrix(matrixFile, useInput, refRel, fromScratch) 00404 except Exception, e: 00405 print "ERROR reading file:", matrixFile, str(e) 00406 raise 00407 00408 try: 00409 self.createWorkFlows(matrixFile) 00410 except Exception, e: 00411 print "ERROR creating workflows :", str(e) 00412 raise 00413
| def MatrixReader::MatrixReader::readMatrix | ( | self, | |
| fileNameIn, | |||
useInput = None, |
|||
refRel = None, |
|||
fromScratch = None |
|||
| ) |
Definition at line 79 of file MatrixReader.py.
00080 : 00081 00082 prefix = self.filesPrefMap[fileNameIn] 00083 00084 print "processing ", fileNameIn 00085 00086 try: 00087 _tmpMod = __import__( 'Configuration.PyReleaseValidation.'+fileNameIn ) 00088 self.relvalModule = sys.modules['Configuration.PyReleaseValidation.'+fileNameIn] 00089 except Exception, e: 00090 print "ERROR importing file ", fileNameIn, str(e) 00091 return 00092 00093 print "request for INPUT for ", useInput 00094 00095 00096 fromInput={} 00097 00098 if useInput: 00099 for i in useInput: 00100 if ':' in i: 00101 (ik,il)=i.split(':') 00102 if ik=='all': 00103 for k in self.relvalModule.workflows.keys(): 00104 fromInput[float(k)]=int(il) 00105 else: 00106 fromInput[float(ik)]=int(il) 00107 else: 00108 if i=='all': 00109 for k in self.relvalModule.workflows.keys(): 00110 fromInput[float(k)]=0 00111 else: 00112 fromInput[float(i)]=0 00113 00114 if fromScratch: 00115 fromScratch=map(float,fromScratch) 00116 for num in fromScratch: 00117 if num in fromInput: 00118 fromInput.pop(num) 00119 #overwrite steps 00120 if self.overWrite: 00121 for p in self.overWrite: 00122 self.relvalModule.steps.overwrite(p) 00123 00124 #change the origin of dataset on the fly 00125 if refRel: 00126 self.relvalModule.changeRefRelease( 00127 self.relvalModule.steps, 00128 [(x,refRel) for x in self.relvalModule.baseDataSetRelease] 00129 ) 00130 00131 00132 for num, wfInfo in self.relvalModule.workflows.items(): 00133 commands=[] 00134 wfName = wfInfo[0] 00135 stepList = wfInfo[1] 00136 # if no explicit name given for the workflow, use the name of step1 00137 if wfName.strip() == '': wfName = stepList[0] 00138 # option to specialize the wf as the third item in the WF list 00139 addTo=None 00140 addCom=None 00141 if len(wfInfo)>=3: 00142 addCom=wfInfo[2] 00143 if not type(addCom)==list: addCom=[addCom] 00144 #print 'added dict',addCom 00145 if len(wfInfo)>=4: 00146 addTo=wfInfo[3] 00147 #pad with 0 00148 while len(addTo)!=len(stepList): 00149 addTo.append(0) 00150 00151 name=wfName 00152 stepIndex=0 00153 ranStepList=[] 00154 00155 #first resolve INPUT possibilities 00156 if num in fromInput: 00157 ilevel=fromInput[num] 00158 #print num,ilevel 00159 for (stepIr,step) in enumerate(reversed(stepList)): 00160 stepName=step 00161 stepI=(len(stepList)-stepIr)-1 00162 #print stepIr,step,stepI,ilevel 00163 if stepI>ilevel: 00164 #print "ignoring" 00165 continue 00166 if stepI!=0: 00167 testName='__'.join(stepList[0:stepI+1])+'INPUT' 00168 else: 00169 testName=step+'INPUT' 00170 #print "JR",stepI,stepIr,testName,stepList 00171 if testName in self.relvalModule.steps.keys(): 00172 #print "JR",stepI,stepIr 00173 stepList[stepI]=testName 00174 #pop the rest in the list 00175 #print "\tmod prepop",stepList 00176 for p in range(stepI): 00177 stepList.pop(0) 00178 #print "\t\tmod",stepList 00179 break 00180 00181 00182 for (stepI,step) in enumerate(stepList): 00183 stepName=step 00184 if self.wm: 00185 #cannot put a certain number of things in wm 00186 if stepName in ['SKIMD','HARVESTD','HARVEST','HARVESTD','RECODFROMRAWRECO']: 00187 continue 00188 #replace stepName is needed 00189 #if stepName in self.replaceStep 00190 if len(name) > 0 : name += '+' 00191 #any step can be mirrored with INPUT 00192 ## maybe we want too level deep input 00193 """ 00194 if num in fromInput: 00195 if step+'INPUT' in self.relvalModule.steps.keys(): 00196 stepName = step+"INPUT" 00197 stepList.remove(step) 00198 stepList.insert(stepIndex,stepName) 00199 """ 00200 name += stepName 00201 00202 if addCom and (not addTo or addTo[stepIndex]==1): 00203 from Configuration.PyReleaseValidation.relval_steps import merge 00204 copyStep=merge(addCom+[self.relvalModule.steps[stepName]]) 00205 cfg, input, opts = self.makeCmd(copyStep) 00206 else: 00207 cfg, input, opts = self.makeCmd(self.relvalModule.steps[stepName]) 00208 00209 if input and cfg : 00210 msg = "FATAL ERROR: found both cfg and input for workflow "+str(num)+' step '+stepName 00211 raise MatrixException(msg) 00212 00213 if input: 00214 cmd = input 00215 else: 00216 if cfg: 00217 cmd = 'cmsDriver.py '+cfg+' '+opts 00218 else: 00219 cmd = 'cmsDriver.py step'+str(stepIndex+1)+' '+opts 00220 if self.wm: 00221 cmd+=' --io %s.io --python %s.py'%(stepName,stepName) 00222 if self.addCommand: 00223 cmd +=' '+self.addCommand 00224 commands.append(cmd) 00225 ranStepList.append(stepName) 00226 stepIndex+=1 00227 00228 self.workFlowSteps[(num,prefix)] = (num, name, commands, ranStepList) 00229 00230 return 00231
| def MatrixReader::MatrixReader::reset | ( | self, | |
what = 'all' |
|||
| ) |
Definition at line 29 of file MatrixReader.py.
00030 : 00031 00032 self.what = what 00033 00034 #a bunch of information, but not yet the WorkFlow object 00035 self.workFlowSteps = {} 00036 #the actual WorkFlow objects 00037 self.workFlows = [] 00038 self.nameList = {} 00039 00040 self.filesPrefMap = {'relval_standard' : 'std-' , 00041 'relval_highstats': 'hi-' , 00042 'relval_pileup': 'PU-' , 00043 'relval_generator': 'gen-' , 00044 'relval_production': 'prod-' , 00045 'relval_ged': 'ged-' 00046 } 00047 00048 self.files = ['relval_standard' , 00049 'relval_highstats', 00050 'relval_pileup', 00051 'relval_generator', 00052 'relval_production', 00053 'relval_ged' 00054 ] 00055 00056 self.relvalModule = None 00057 00058 return
| def MatrixReader::MatrixReader::show | ( | self, | |
selected = None, |
|||
extended = True |
|||
| ) |
Definition at line 414 of file MatrixReader.py.
00414 : 00415 00416 self.showWorkFlows(selected,extended) 00417 print '\n','-'*80,'\n' 00418 00419
| def MatrixReader::MatrixReader::showRaw | ( | self, | |
| useInput, | |||
refRel = None, |
|||
fromScratch = None, |
|||
what = 'all', |
|||
step1Only = False, |
|||
selected = None |
|||
| ) |
Definition at line 232 of file MatrixReader.py.
00233 : 00234 00235 if selected: 00236 selected=map(float,selected) 00237 for matrixFile in self.files: 00238 00239 self.reset(what) 00240 00241 if self.what != 'all' and self.what not in matrixFile: 00242 print "ignoring non-requested file",matrixFile 00243 continue 00244 00245 try: 00246 self.readMatrix(matrixFile, useInput, refRel, fromScratch) 00247 except Exception, e: 00248 print "ERROR reading file:", matrixFile, str(e) 00249 raise 00250 00251 if not self.workFlowSteps: continue 00252 00253 dataFileName = matrixFile.replace('relval_', 'cmsDriver_')+'_hlt.txt' 00254 outFile = open(dataFileName,'w') 00255 00256 print "found ", len(self.workFlowSteps.keys()), ' workflows for ', dataFileName 00257 ids = self.workFlowSteps.keys() 00258 ids.sort() 00259 indexAndSteps=[] 00260 00261 writtenWF=0 00262 for key in ids: 00263 if selected and not (key[0] in selected): 00264 continue 00265 #trick to skip the HImix IB test 00266 if key[0]==203.1 or key[0]==204.1 or key[0]==205.1 or key[0]==4.51 or key[0]==4.52: continue 00267 num, name, commands, stepList = self.workFlowSteps[key] 00268 00269 wfName,stepNames= name.split('+',1) 00270 00271 stepNames=stepNames.replace('+RECODFROMRAWRECO','') 00272 stepNames=stepNames.replace('+SKIMCOSD','') 00273 stepNames=stepNames.replace('+SKIMD','') 00274 if 'HARVEST' in stepNames: 00275 #find out automatically what to remove 00276 exactb=stepNames.index('+HARVEST') 00277 exacte=stepNames.index('+',exactb+1) if ('+' in stepNames[exactb+1:]) else (len(stepNames)) 00278 stepNames=stepNames.replace(stepNames[exactb:exacte],'') 00279 otherSteps = None 00280 if '+' in stepNames: 00281 step1,otherSteps = stepNames.split('+',1) 00282 00283 line = str(num) + ' ++ '+ wfName 00284 if otherSteps and not step1Only: 00285 line += ' ++ ' +otherSteps.replace('+',',') 00286 else: 00287 line += ' ++ none' 00288 inputInfo=None 00289 if not isinstance(commands[0],str): 00290 inputInfo=commands[0] 00291 if otherSteps: 00292 for (i,c) in enumerate(otherSteps.split('+')): 00293 #pad with set 00294 for p in range(len(indexAndSteps),i+2): 00295 indexAndSteps.append(set()) 00296 indexAndSteps[i+1].add((c,commands[i+1])) 00297 00298 if inputInfo : 00299 #skip the samples from INPUT when step1Only is on 00300 if step1Only: continue 00301 line += ' ++ REALDATA: '+inputInfo.dataSet 00302 if inputInfo.run!=[]: line += ', RUN:'+'|'.join(map(str,inputInfo.run)) 00303 line += ', FILES: ' +str(inputInfo.files) 00304 line += ', EVENTS: '+str(inputInfo.events) 00305 if inputInfo.label!='': 00306 line += ', LABEL: ' +inputInfo.label 00307 line += ', LOCATION:'+inputInfo.location 00308 line += ' @@@' 00309 else: 00310 line += ' @@@ '+commands[0] 00311 line=line.replace('DQMROOT','DQM') 00312 writtenWF+=1 00313 outFile.write(line+'\n') 00314 00315 00316 outFile.write('\n'+'\n') 00317 if step1Only: continue 00318 00319 for (index,s) in enumerate(indexAndSteps): 00320 for (stepName,cmd) in s: 00321 stepIndex=index+1 00322 if 'dbsquery.log' in cmd: continue 00323 line = 'STEP%d ++ '%(stepIndex,) +stepName + ' @@@ '+cmd 00324 line=line.replace('DQMROOT','DQM') 00325 outFile.write(line+'\n') 00326 outFile.write('\n'+'\n') 00327 outFile.close() 00328 print "wrote ",writtenWF, ' workflow'+('s' if (writtenWF!=1) else ''),' to ', outFile.name 00329 return 00330
| def MatrixReader::MatrixReader::showWorkFlows | ( | self, | |
selected = None, |
|||
extended = True |
|||
| ) |
Definition at line 331 of file MatrixReader.py.
00332 : 00333 if selected: selected = map(float,selected) 00334 maxLen = 100 # for summary, limit width of output 00335 fmt1 = "%-6s %-35s [1]: %s ..." 00336 fmt2 = " %35s [%d]: %s ..." 00337 print "\nfound a total of ", len(self.workFlows), ' workflows:' 00338 if selected: 00339 print " of which the following", len(selected), 'were selected:' 00340 #-ap for now: 00341 maxLen = -1 # for individual listing, no limit on width 00342 fmt1 = "%-6s %-35s [1]: %s " 00343 fmt2 = " %35s [%d]: %s" 00344 00345 N=[] 00346 for wf in self.workFlows: 00347 if selected and float(wf.numId) not in selected: continue 00348 if extended: print '' 00349 #pad with zeros 00350 for i in range(len(N),len(wf.cmds)): N.append(0) 00351 N[len(wf.cmds)-1]+=1 00352 wfName, stepNames = wf.nameId.split('+',1) 00353 for i,s in enumerate(wf.cmds): 00354 if extended: 00355 if i==0: 00356 print fmt1 % (wf.numId, stepNames, (str(s)+' ')[:maxLen]) 00357 else: 00358 print fmt2 % ( ' ', i+1, (str(s)+' ')[:maxLen]) 00359 else: 00360 print "%-6s %-35s "% (wf.numId, stepNames) 00361 break 00362 print '' 00363 for i,n in enumerate(N): 00364 if n: print n,'workflows with',i+1,'steps' 00365 00366 return
| def MatrixReader::MatrixReader::updateDB | ( | self | ) |
Definition at line 420 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
1.7.3