Public Member Functions
def	__init__
def	createWorkFlows
def	makeCmd
def	prepare
def	readMatrix
def	reset
def	show
def	showRaw
def	showWorkFlows
def	updateDB
Public Attributes
	files
	filesPrefMap
	nameList
	relvalModule
	step1WorkFlows
	step2WorkFlows
	step3WorkFlows
	step4WorkFlows
	workFlows

Detailed Description

Definition at line 254 of file runTheMatrix_dev.py.

Constructor & Destructor Documentation

def runTheMatrix_dev::MatrixReader::__init__ ( self )

Definition at line 256 of file runTheMatrix_dev.py.

00257                       :
00258 
00259         self.reset()
00260 
00261         return

Member Function Documentation

def runTheMatrix_dev::MatrixReader::createWorkFlows	(	self,
		fileNameIn
	)

Definition at line 466 of file runTheMatrix_dev.py.

00467                                          :
00468 
00469         prefixIn = self.filesPrefMap[fileNameIn]
00470 
00471         # get through the list of items and update the requested workflows only
00472         keyList = self.step1WorkFlows.keys()
00473         ids = []
00474         for item in keyList:
00475             id, pref = item
00476             if pref != prefixIn : continue
00477             ids.append( float(id) )
00478             
00479         ids.sort()
00480         n1 = 0
00481         n2 = 0
00482         n3 = 0
00483         n4 = 0
00484         for key in ids:
00485             val = self.step1WorkFlows[(key,prefixIn)]
00486             num, name, cmd, step2, step3, step4, inputInfo = val
00487             nameId = num+'_'+name
00488             if nameId in self.nameList.keys():
00489                 print "==> duplicate name found for ", nameId
00490                 print '    keeping  : ', self.nameList[nameId]
00491                 print '    ignoring : ', val
00492             else:
00493                 self.nameList[nameId] = val
00494 
00495             cmd2 = None
00496             cmd3 = None
00497             cmd4 = None
00498             
00499             n1 += 1
00500 
00501             if step2.lower() != '':
00502                 n2 += 1
00503                 cmd2 = step2
00504                 if step3.lower() != '':
00505                     n3 += 1
00506                     cmd3 = step3
00507                     if step4.lower() != '':
00508                         n4 += 1
00509                         cmd4 = step4
00510                     #print '\tstep3 : ', self.step3WorkFlows[step3]
00511             self.workFlows.append( WorkFlow(num, name, cmd, cmd2, cmd3, cmd4, inputInfo) )
00512 
00513         return

def runTheMatrix_dev::MatrixReader::makeCmd	(	self,
		step
	)

Definition at line 286 of file runTheMatrix_dev.py.

00287                            :
00288 
00289         cmd = ''
00290         cfg = None
00291         input = None
00292         #print step
00293         #print defaults
00294         for k,v in step.items():
00295             if 'no_exec' in k : continue  # we want to really run it ...
00296             if k.lower() == 'cfg':
00297                 cfg = v
00298                 continue # do not append to cmd, return separately
00299             if k.lower() == 'input':
00300                 input = v
00301                 continue # do not append to cmd, return separately
00302             #print k,v
00303             cmd += ' ' + k + ' ' + str(v)
00304         return cfg, input, cmd

def runTheMatrix_dev::MatrixReader::prepare	(	self,
		useInput = `None`
	)

Definition at line 514 of file runTheMatrix_dev.py.

00515                                     :
00516         
00517         for matrixFile in self.files:
00518             try:
00519                 self.readMatrix(matrixFile, useInput)
00520             except Exception, e:
00521                 print "ERROR reading file:", matrixFile, str(e)
00522                 raise
00523 
00524             try:
00525                 self.createWorkFlows(matrixFile)
00526             except Exception, e:
00527                 print "ERROR creating workflows :", str(e)
00528                 raise

def runTheMatrix_dev::MatrixReader::readMatrix	(	self,
		fileNameIn,
		useInput = `None`
	)

Definition at line 305 of file runTheMatrix_dev.py.

00306                                                    :
00307         
00308         prefix = self.filesPrefMap[fileNameIn]
00309 
00310         print "processing ", fileNameIn
00311 
00312         try:
00313             _tmpMod = __import__( 'Configuration.PyReleaseValidation.'+fileNameIn )
00314             self.relvalModule = sys.modules['Configuration.PyReleaseValidation.'+fileNameIn]
00315         except Exception, e:
00316             print "ERROR importing file ", fileNameIn, str(e)
00317             return
00318 
00319         print "request for INPUT for ", useInput
00320 
00321         for num, wfInfo in self.relvalModule.workflows.items():
00322             wfName = wfInfo[0]
00323             stepList = wfInfo[1]
00324             # if no explicit name given for the workflow, use the name of step1
00325             if wfName.strip() == '': wfName = stepList[0] 
00326             stepCmds = ['','','','']
00327             stepIndex = 0
00328             name  = wfName
00329             inputInfo = None
00330             for step in stepList:
00331                 if len(name) > 0 : name += '+'
00332                 stepName = step
00333                 if stepIndex==0 and useInput and (str(num) in useInput or "all" in useInput):
00334                     # print "--> using INPUT as step1 for workflow ", num
00335                     if step+'INPUT' in self.relvalModule.step1.keys():
00336                         stepName = step+"INPUT"
00337                 name += stepName
00338                 cfg, input, opts = self.makeCmd(self.relvalModule.stepList[stepIndex][stepName])
00339                 if input and cfg :
00340                     msg = "FATAL ERROR: found both cfg and input for workflow "+str(num)+' step '+stepName
00341                     raise msg
00342 
00343                 if cfg:
00344                     cmd  = 'cmsDriver.py '+cfg+' '+opts
00345                 if stepIndex==0 and not inputInfo and input: # only if we didn't already set the input
00346                     inputInfo = input
00347                     cmd = 'DATAINPUT from '+inputInfo.dataSet
00348                     
00349                 if stepIndex > 0:
00350                     cmd  = 'cmsDriver.py step'+str(stepIndex+1)+'.py '+opts
00351                     
00352                 stepCmds[stepIndex] = cmd
00353                 stepIndex += 1
00354 
00355             self.step1WorkFlows[(float(num),prefix)] = (str(float(num)), name, stepCmds[0], stepCmds[1], stepCmds[2], stepCmds[3], inputInfo)
00356         
00357         return

def runTheMatrix_dev::MatrixReader::reset ( self )

Definition at line 262 of file runTheMatrix_dev.py.

00263                    :
00264 
00265         self.step1WorkFlows = {}
00266         self.step2WorkFlows = {}
00267         self.step3WorkFlows = {}
00268         self.step4WorkFlows = {}
00269 
00270         self.workFlows = []
00271         self.nameList  = {}
00272         
00273         self.filesPrefMap = {'relval_standard' : 'std-' ,
00274                              'relval_highstats': 'hi-'  ,
00275                              'relval_generator': 'gen-'  ,
00276                              }
00277 
00278         self.files = ['relval_standard' ,
00279                       'relval_highstats',
00280                       'relval_generator',
00281                       ]
00282 
00283         self.relvalModule = None
00284         
00285         return

def runTheMatrix_dev::MatrixReader::show	(	self,
		selected = `None`
	)

Definition at line 529 of file runTheMatrix_dev.py.

00529                                  :    
00530         # self.showRaw()
00531         self.showWorkFlows(selected)
00532         print '\n','-'*80,'\n'
00533 
00534

def runTheMatrix_dev::MatrixReader::showRaw	(	self,
		useInput
	)

Definition at line 358 of file runTheMatrix_dev.py.

00359                                :
00360 
00361         for matrixFile in self.files:
00362             self.reset()
00363             try:
00364                 self.readMatrix(matrixFile, useInput)
00365             except Exception, e:
00366                 print "ERROR reading file:", matrixFile, str(e)
00367                 raise
00368 
00369             if not self.step1WorkFlows: continue
00370 
00371             dataFileName = matrixFile.replace('relval_', 'cmsDriver_')+'_hlt.txt'
00372             outFile = open(dataFileName,'w')
00373 
00374             print "found ", len(self.step1WorkFlows.keys()), ' workflows for ', dataFileName
00375             ids = self.step1WorkFlows.keys()
00376             ids.sort()
00377             stepCmds = ['','','','']
00378             for key in ids:
00379                 num, name, stepCmds[0], stepCmds[1], stepCmds[2], stepCmds[3], inputInfo = self.step1WorkFlows[key]
00380                 wfName,stepNames= name.split('+',1)
00381                 otherSteps = None
00382                 if '+' in stepNames:
00383                     step1,otherSteps = stepNames.split('+',1)
00384                 line = num + ' ++ '+ wfName 
00385                 if otherSteps:
00386                     line += ' ++ ' +otherSteps.replace('+',',')
00387                 else:
00388                     line += ' ++ none' 
00389                 if inputInfo :
00390                     line += ' ++ REALDATA: '+inputInfo.dataSet
00391                     line += ', FILES: ' +str(inputInfo.files)
00392                     line += ', EVENTS: '+str(inputInfo.events)
00393                     line += ', LABEL: ' +inputInfo.label
00394                     line += ', LOCATION:'+inputInfo.location
00395                     line += ' @@@'
00396                 else:
00397                     line += ' @@@ '+stepCmds[0]
00398                 print line
00399                 outFile.write(line+'\n')
00400 
00401             outFile.write('\n'+'\n')
00402             for stepName in self.relvalModule.step2.keys():
00403                 cfg,input,cmd = self.makeCmd(self.relvalModule.step2[stepName])
00404                 line = 'STEP2 ++ ' +stepName + ' @@@ cmsDriver.py step2 ' +cmd
00405                 print line
00406                 outFile.write(line+'\n')
00407                 
00408             outFile.write('\n'+'\n')
00409             for stepName in self.relvalModule.step3.keys():
00410                 cfg,input,cmd = self.makeCmd(self.relvalModule.step3[stepName])
00411                 line ='STEP3 ++ ' +stepName + ' @@@ cmsDriver.py step3_RELVAL ' +cmd 
00412                 print line
00413                 outFile.write(line+'\n')
00414                 
00415             outFile.write('\n'+'\n')
00416             for stepName in self.relvalModule.step4.keys():
00417                 cfg,input,cmd = self.makeCmd(self.relvalModule.step4[stepName])
00418                 line = 'STEP4 ++ ' +stepName + ' @@@ cmsDriver.py step4 ' +cmd
00419                 print line
00420                 outFile.write(line+'\n')
00421                 
00422             outFile.close()
00423 
00424         
00425         return

def runTheMatrix_dev::MatrixReader::showWorkFlows	(	self,
		selected = `None`
	)

Definition at line 426 of file runTheMatrix_dev.py.

00427                                           :
00428 
00429         maxLen = 100 # for summary, limit width of output
00430         fmt1   = "%-6s %-35s [1]: %s ..."
00431         fmt2   = "       %35s [%d]: %s ..."
00432         print "\nfound a total of ", len(self.workFlows), ' workflows:'
00433         if selected:
00434             print "      of which the following", len(selected), 'were selected:'
00435         #-ap for now:
00436         maxLen = -1  # for individual listing, no limit on width
00437         fmt1   = "%-6s %-35s [1]: %s " 
00438         fmt2   = "       %35s [%d]: %s"
00439 
00440         n1 = 0
00441         n2 = 0
00442         n3 = 0
00443         n4 = 0
00444         for wf in self.workFlows:
00445             if selected and float(wf.numId) not in selected: continue
00446             print ''
00447             n1+=1
00448             wfName, stepNames = wf.nameId.split('+',1)
00449             print fmt1 % (wf.numId, stepNames, (wf.cmdStep1+' ')[:maxLen])
00450             if wf.cmdStep2:
00451                 n2+=1
00452                 print fmt2 % ( ' ', 2, (wf.cmdStep2+' ')[:maxLen])
00453                 if wf.cmdStep3:
00454                     n3+=1
00455                     print fmt2 % ( ' ', 3, (wf.cmdStep3+' ')[:maxLen])
00456                     if wf.cmdStep4:
00457                         n4+=1
00458                         print fmt2 % ( ' ', 4, (wf.cmdStep4+' ')[:maxLen])
00459 
00460         print n1, 'workflows for step1,'
00461         print n2, 'workflows for step1 + step2,'
00462         print n3, 'workflows for step1 + step2 + step3'
00463         print n4, 'workflows for step1 + step2 + step3 + step4'
00464 
00465         return

def runTheMatrix_dev::MatrixReader::updateDB ( self )

Definition at line 535 of file runTheMatrix_dev.py.

00536                       :
00537 
00538         import pickle
00539         pickle.dump(self.workFlows, open('theMatrix.pkl', 'w') )
00540 
00541         return
00542 
00543 # ================================================================================