|
|
|
Public Member Functions | |
| def | __init__ |
| def | createWorkFlows |
| def | makeCmd |
| def | prepare |
| def | readMatrix |
| def | reset |
| def | show |
| def | showRaw |
| def | showWorkFlows |
| def | updateDB |
Public Attributes | |
| addCommand | |
| commandLineWf | |
| files | |
| filesPrefMap | |
| nameList | |
| overWrite | |
| relvalModule | |
| what | |
| wm | |
| workFlows | |
| workFlowSteps | |
Definition at line 16 of file MatrixReader.py.
| def MatrixReader::MatrixReader::__init__ | ( | self, | |
| opt | |||
| ) |
Definition at line 18 of file MatrixReader.py.
| def MatrixReader::MatrixReader::createWorkFlows | ( | self, | |
| fileNameIn | |||
| ) |
Definition at line 370 of file MatrixReader.py.
00371 : 00372 00373 prefixIn = self.filesPrefMap[fileNameIn] 00374 00375 # get through the list of items and update the requested workflows only 00376 keyList = self.workFlowSteps.keys() 00377 ids = [] 00378 for item in keyList: 00379 id, pref = item 00380 if pref != prefixIn : continue 00381 ids.append(id) 00382 ids.sort() 00383 for key in ids: 00384 val = self.workFlowSteps[(key,prefixIn)] 00385 num, name, commands, stepList = val 00386 nameId = str(num)+'_'+name 00387 if nameId in self.nameList: 00388 print "==> duplicate name found for ", nameId 00389 print ' keeping : ', self.nameList[nameId] 00390 print ' ignoring : ', val 00391 else: 00392 self.nameList[nameId] = val 00393 00394 self.workFlows.append(WorkFlow(num, name, commands=commands)) 00395 00396 return
| def MatrixReader::MatrixReader::makeCmd | ( | self, | |
| step | |||
| ) |
Definition at line 59 of file MatrixReader.py.
00060 : 00061 00062 cmd = '' 00063 cfg = None 00064 input = None 00065 for k,v in step.items(): 00066 if 'no_exec' in k : continue # we want to really run it ... 00067 if k.lower() == 'cfg': 00068 cfg = v 00069 continue # do not append to cmd, return separately 00070 if k.lower() == 'input': 00071 input = v 00072 continue # do not append to cmd, return separately 00073 00074 #chain the configs 00075 #if k.lower() == '--python': 00076 # v = 'step%d_%s'%(index,v) 00077 cmd += ' ' + k + ' ' + str(v) 00078 return cfg, input, cmd
| def MatrixReader::MatrixReader::prepare | ( | self, | |
useInput = None, |
|||
refRel = '', |
|||
fromScratch = None |
|||
| ) |
Definition at line 397 of file MatrixReader.py.
00398 : 00399 00400 for matrixFile in self.files: 00401 if self.what != 'all' and self.what not in matrixFile: 00402 print "ignoring non-requested file",matrixFile 00403 continue 00404 00405 try: 00406 self.readMatrix(matrixFile, useInput, refRel, fromScratch) 00407 except Exception, e: 00408 print "ERROR reading file:", matrixFile, str(e) 00409 raise 00410 00411 try: 00412 self.createWorkFlows(matrixFile) 00413 except Exception, e: 00414 print "ERROR creating workflows :", str(e) 00415 raise 00416
| def MatrixReader::MatrixReader::readMatrix | ( | self, | |
| fileNameIn, | |||
useInput = None, |
|||
refRel = None, |
|||
fromScratch = None |
|||
| ) |
Definition at line 79 of file MatrixReader.py.
00080 : 00081 00082 prefix = self.filesPrefMap[fileNameIn] 00083 00084 print "processing ", fileNameIn 00085 00086 try: 00087 _tmpMod = __import__( 'Configuration.PyReleaseValidation.'+fileNameIn ) 00088 self.relvalModule = sys.modules['Configuration.PyReleaseValidation.'+fileNameIn] 00089 except Exception, e: 00090 print "ERROR importing file ", fileNameIn, str(e) 00091 return 00092 00093 print "request for INPUT for ", useInput 00094 00095 00096 fromInput={} 00097 00098 if useInput: 00099 for i in useInput: 00100 if ':' in i: 00101 (ik,il)=i.split(':') 00102 if ik=='all': 00103 for k in self.relvalModule.workflows.keys(): 00104 fromInput[float(k)]=int(il) 00105 else: 00106 fromInput[float(ik)]=int(il) 00107 else: 00108 if i=='all': 00109 for k in self.relvalModule.workflows.keys(): 00110 fromInput[float(k)]=0 00111 else: 00112 fromInput[float(i)]=0 00113 00114 if fromScratch: 00115 fromScratch=map(float,fromScratch) 00116 for num in fromScratch: 00117 if num in fromInput: 00118 fromInput.pop(num) 00119 #overwrite steps 00120 if self.overWrite: 00121 for p in self.overWrite: 00122 self.relvalModule.steps.overwrite(p) 00123 00124 #change the origin of dataset on the fly 00125 if refRel: 00126 self.relvalModule.changeRefRelease( 00127 self.relvalModule.steps, 00128 [(x,refRel) for x in self.relvalModule.baseDataSetRelease] 00129 ) 00130 00131 00132 for num, wfInfo in self.relvalModule.workflows.items(): 00133 commands=[] 00134 wfName = wfInfo[0] 00135 stepList = wfInfo[1] 00136 # if no explicit name given for the workflow, use the name of step1 00137 if wfName.strip() == '': wfName = stepList[0] 00138 # option to specialize the wf as the third item in the WF list 00139 addTo=None 00140 addCom=None 00141 if len(wfInfo)>=3: 00142 addCom=wfInfo[2] 00143 if not type(addCom)==list: addCom=[addCom] 00144 #print 'added dict',addCom 00145 if len(wfInfo)>=4: 00146 addTo=wfInfo[3] 00147 #pad with 0 00148 while len(addTo)!=len(stepList): 00149 addTo.append(0) 00150 00151 name=wfName 00152 stepIndex=0 00153 ranStepList=[] 00154 00155 #first resolve INPUT possibilities 00156 if num in fromInput: 00157 ilevel=fromInput[num] 00158 #print num,ilevel 00159 for (stepIr,step) in enumerate(reversed(stepList)): 00160 stepName=step 00161 stepI=(len(stepList)-stepIr)-1 00162 #print stepIr,step,stepI,ilevel 00163 if stepI>ilevel: 00164 #print "ignoring" 00165 continue 00166 if stepI!=0: 00167 testName='__'.join(stepList[0:stepI+1])+'INPUT' 00168 else: 00169 testName=step+'INPUT' 00170 #print "JR",stepI,stepIr,testName,stepList 00171 if testName in self.relvalModule.steps.keys(): 00172 #print "JR",stepI,stepIr 00173 stepList[stepI]=testName 00174 #pop the rest in the list 00175 #print "\tmod prepop",stepList 00176 for p in range(stepI): 00177 stepList.pop(0) 00178 #print "\t\tmod",stepList 00179 break 00180 00181 00182 for (stepI,step) in enumerate(stepList): 00183 stepName=step 00184 if self.wm: 00185 #cannot put a certain number of things in wm 00186 if stepName in ['HARVEST','HARVESTD','HARVESTDreHLT','RECODFROMRAWRECO','SKIMD','SKIMCOSD','SKIMDreHLT']: 00187 continue 00188 00189 #replace stepName is needed 00190 #if stepName in self.replaceStep 00191 if len(name) > 0 : name += '+' 00192 #any step can be mirrored with INPUT 00193 ## maybe we want too level deep input 00194 """ 00195 if num in fromInput: 00196 if step+'INPUT' in self.relvalModule.steps.keys(): 00197 stepName = step+"INPUT" 00198 stepList.remove(step) 00199 stepList.insert(stepIndex,stepName) 00200 """ 00201 name += stepName 00202 00203 if addCom and (not addTo or addTo[stepIndex]==1): 00204 from Configuration.PyReleaseValidation.relval_steps import merge 00205 copyStep=merge(addCom+[self.relvalModule.steps[stepName]]) 00206 cfg, input, opts = self.makeCmd(copyStep) 00207 else: 00208 cfg, input, opts = self.makeCmd(self.relvalModule.steps[stepName]) 00209 00210 if input and cfg : 00211 msg = "FATAL ERROR: found both cfg and input for workflow "+str(num)+' step '+stepName 00212 raise MatrixException(msg) 00213 00214 if input: 00215 cmd = input 00216 else: 00217 if cfg: 00218 cmd = 'cmsDriver.py '+cfg+' '+opts 00219 else: 00220 cmd = 'cmsDriver.py step'+str(stepIndex+1)+' '+opts 00221 if self.wm: 00222 cmd+=' --io %s.io --python %s.py'%(stepName,stepName) 00223 if self.addCommand: 00224 cmd +=' '+self.addCommand 00225 if self.wm: 00226 cmd=cmd.replace('DQMROOT','DQM') 00227 commands.append(cmd) 00228 ranStepList.append(stepName) 00229 stepIndex+=1 00230 00231 self.workFlowSteps[(num,prefix)] = (num, name, commands, ranStepList) 00232 00233 return 00234
| def MatrixReader::MatrixReader::reset | ( | self, | |
what = 'all' |
|||
| ) |
Definition at line 29 of file MatrixReader.py.
00030 : 00031 00032 self.what = what 00033 00034 #a bunch of information, but not yet the WorkFlow object 00035 self.workFlowSteps = {} 00036 #the actual WorkFlow objects 00037 self.workFlows = [] 00038 self.nameList = {} 00039 00040 self.filesPrefMap = {'relval_standard' : 'std-' , 00041 'relval_highstats': 'hi-' , 00042 'relval_pileup': 'PU-' , 00043 'relval_generator': 'gen-' , 00044 'relval_production': 'prod-' , 00045 #'relval_ged': 'ged-' 00046 } 00047 00048 self.files = ['relval_standard' , 00049 'relval_highstats', 00050 'relval_pileup', 00051 'relval_generator', 00052 'relval_production', 00053 #'relval_ged' 00054 ] 00055 00056 self.relvalModule = None 00057 00058 return
| def MatrixReader::MatrixReader::show | ( | self, | |
selected = None, |
|||
extended = True |
|||
| ) |
Definition at line 417 of file MatrixReader.py.
00417 : 00418 00419 self.showWorkFlows(selected,extended) 00420 print '\n','-'*80,'\n' 00421 00422
| def MatrixReader::MatrixReader::showRaw | ( | self, | |
| useInput, | |||
refRel = None, |
|||
fromScratch = None, |
|||
what = 'all', |
|||
step1Only = False, |
|||
selected = None |
|||
| ) |
Definition at line 235 of file MatrixReader.py.
00236 : 00237 00238 if selected: 00239 selected=map(float,selected) 00240 for matrixFile in self.files: 00241 00242 self.reset(what) 00243 00244 if self.what != 'all' and self.what not in matrixFile: 00245 print "ignoring non-requested file",matrixFile 00246 continue 00247 00248 try: 00249 self.readMatrix(matrixFile, useInput, refRel, fromScratch) 00250 except Exception, e: 00251 print "ERROR reading file:", matrixFile, str(e) 00252 raise 00253 00254 if not self.workFlowSteps: continue 00255 00256 dataFileName = matrixFile.replace('relval_', 'cmsDriver_')+'_hlt.txt' 00257 outFile = open(dataFileName,'w') 00258 00259 print "found ", len(self.workFlowSteps.keys()), ' workflows for ', dataFileName 00260 ids = self.workFlowSteps.keys() 00261 ids.sort() 00262 indexAndSteps=[] 00263 00264 writtenWF=0 00265 for key in ids: 00266 if selected and not (key[0] in selected): 00267 continue 00268 #trick to skip the HImix IB test 00269 if key[0]==203.1 or key[0]==204.1 or key[0]==205.1 or key[0]==4.51 or key[0]==4.52: continue 00270 num, name, commands, stepList = self.workFlowSteps[key] 00271 00272 wfName,stepNames= name.split('+',1) 00273 00274 stepNames=stepNames.replace('+RECODFROMRAWRECO','') 00275 stepNames=stepNames.replace('+SKIMCOSD','') 00276 stepNames=stepNames.replace('+SKIMD','') 00277 if 'HARVEST' in stepNames: 00278 #find out automatically what to remove 00279 exactb=stepNames.index('+HARVEST') 00280 exacte=stepNames.index('+',exactb+1) if ('+' in stepNames[exactb+1:]) else (len(stepNames)) 00281 stepNames=stepNames.replace(stepNames[exactb:exacte],'') 00282 otherSteps = None 00283 if '+' in stepNames: 00284 step1,otherSteps = stepNames.split('+',1) 00285 00286 line = str(num) + ' ++ '+ wfName 00287 if otherSteps and not step1Only: 00288 line += ' ++ ' +otherSteps.replace('+',',') 00289 else: 00290 line += ' ++ none' 00291 inputInfo=None 00292 if not isinstance(commands[0],str): 00293 inputInfo=commands[0] 00294 if otherSteps: 00295 for (i,c) in enumerate(otherSteps.split('+')): 00296 #pad with set 00297 for p in range(len(indexAndSteps),i+2): 00298 indexAndSteps.append(set()) 00299 indexAndSteps[i+1].add((c,commands[i+1])) 00300 00301 if inputInfo : 00302 #skip the samples from INPUT when step1Only is on 00303 if step1Only: continue 00304 line += ' ++ REALDATA: '+inputInfo.dataSet 00305 if inputInfo.run!=[]: line += ', RUN:'+'|'.join(map(str,inputInfo.run)) 00306 line += ', FILES: ' +str(inputInfo.files) 00307 line += ', EVENTS: '+str(inputInfo.events) 00308 if inputInfo.label!='': 00309 line += ', LABEL: ' +inputInfo.label 00310 line += ', LOCATION:'+inputInfo.location 00311 line += ' @@@' 00312 else: 00313 line += ' @@@ '+commands[0] 00314 line=line.replace('DQMROOT','DQM') 00315 writtenWF+=1 00316 outFile.write(line+'\n') 00317 00318 00319 outFile.write('\n'+'\n') 00320 if step1Only: continue 00321 00322 for (index,s) in enumerate(indexAndSteps): 00323 for (stepName,cmd) in s: 00324 stepIndex=index+1 00325 if 'dbsquery.log' in cmd: continue 00326 line = 'STEP%d ++ '%(stepIndex,) +stepName + ' @@@ '+cmd 00327 line=line.replace('DQMROOT','DQM') 00328 outFile.write(line+'\n') 00329 outFile.write('\n'+'\n') 00330 outFile.close() 00331 print "wrote ",writtenWF, ' workflow'+('s' if (writtenWF!=1) else ''),' to ', outFile.name 00332 return 00333
| def MatrixReader::MatrixReader::showWorkFlows | ( | self, | |
selected = None, |
|||
extended = True |
|||
| ) |
Definition at line 334 of file MatrixReader.py.
00335 : 00336 if selected: selected = map(float,selected) 00337 maxLen = 100 # for summary, limit width of output 00338 fmt1 = "%-6s %-35s [1]: %s ..." 00339 fmt2 = " %35s [%d]: %s ..." 00340 print "\nfound a total of ", len(self.workFlows), ' workflows:' 00341 if selected: 00342 print " of which the following", len(selected), 'were selected:' 00343 #-ap for now: 00344 maxLen = -1 # for individual listing, no limit on width 00345 fmt1 = "%-6s %-35s [1]: %s " 00346 fmt2 = " %35s [%d]: %s" 00347 00348 N=[] 00349 for wf in self.workFlows: 00350 if selected and float(wf.numId) not in selected: continue 00351 if extended: print '' 00352 #pad with zeros 00353 for i in range(len(N),len(wf.cmds)): N.append(0) 00354 N[len(wf.cmds)-1]+=1 00355 wfName, stepNames = wf.nameId.split('+',1) 00356 for i,s in enumerate(wf.cmds): 00357 if extended: 00358 if i==0: 00359 print fmt1 % (wf.numId, stepNames, (str(s)+' ')[:maxLen]) 00360 else: 00361 print fmt2 % ( ' ', i+1, (str(s)+' ')[:maxLen]) 00362 else: 00363 print "%-6s %-35s "% (wf.numId, stepNames) 00364 break 00365 print '' 00366 for i,n in enumerate(N): 00367 if n: print n,'workflows with',i+1,'steps' 00368 00369 return
| def MatrixReader::MatrixReader::updateDB | ( | self | ) |
Definition at line 423 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 18 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
Definition at line 29 of file MatrixReader.py.
1.7.3