Public Member Functions
def	__init__
def	createWorkFlows
def	prepare
def	readMatrix
def	reset
def	show
def	showRaw
def	showWorkFlows
def	updateDB
Public Attributes
	files
	filesPrefMap
	nameList
	step1WorkFlows
	step2WorkFlows
	step3WorkFlows
	step4WorkFlows
	workFlows

Detailed Description

Definition at line 252 of file runTheMatrix.py.

Constructor & Destructor Documentation

def runTheMatrix::MatrixReader::__init__ ( self )

Definition at line 254 of file runTheMatrix.py.

00255                       :
00256 
00257         self.reset()
00258 
00259         return

Member Function Documentation

def runTheMatrix::MatrixReader::createWorkFlows	(	self,
		fileNameIn
	)

Definition at line 450 of file runTheMatrix.py.

00451                                          :
00452 
00453         prefixIn = self.filesPrefMap[fileNameIn]
00454 
00455         # get through the list of items and update the requested workflows only
00456         keyList = self.step1WorkFlows.keys()
00457         ids = []
00458         for item in keyList:
00459             id, pref = item
00460             if pref != prefixIn : continue
00461             ids.append( float(id) )
00462             
00463         ids.sort()
00464         n1 = 0
00465         n2 = 0
00466         n3 = 0
00467         n4 = 0
00468         for key in ids:
00469             val = self.step1WorkFlows[(key,prefixIn)]
00470             num, name, step2, step3, step4, cmd, real = val
00471             nameId = num+'_'+name
00472             if step2.lower() != 'none':
00473                 name += '+'+step2
00474                 if step3.lower() != 'none':
00475                     name += '+'+step3
00476                     if step4.lower() != 'none':
00477                         name += '+'+step4
00478             if nameId in self.nameList.keys():
00479                 print "==> duplicate name found for ", nameId
00480                 print '    keeping  : ', self.nameList[nameId]
00481                 print '    ignoring : ', val
00482             else:
00483                 self.nameList[nameId] = val
00484 
00485             cmd2 = None
00486             cmd3 = None
00487             cmd4 = None
00488             
00489             n1 += 1
00490 
00491             if step2.lower() != 'none':
00492                 n2 += 1
00493                 cmd2 = self.step2WorkFlows[step2]
00494                 if step3.lower() != 'none':
00495                     n3 += 1
00496                     cmd3 = self.step3WorkFlows[step3]
00497                     if step4.lower() != 'none':
00498                         n4 += 1
00499                         cmd4 = self.step4WorkFlows[step4]
00500                     #print '\tstep3 : ', self.step3WorkFlows[step3]
00501             self.workFlows.append( WorkFlow(num, name, cmd, cmd2, cmd3, cmd4) )
00502 
00503         return

def runTheMatrix::MatrixReader::prepare ( self )

Definition at line 504 of file runTheMatrix.py.

00505                      :
00506         
00507         for matrixFile in self.files:
00508             try:
00509                 self.readMatrix(matrixFile)
00510             except Exception, e:
00511                 print "ERROR reading file:", matrixFile, str(e)
00512 
00513             try:
00514                 self.createWorkFlows(matrixFile)
00515             except Exception, e:
00516                 print "ERROR creating workflows :", str(e)

def runTheMatrix::MatrixReader::readMatrix	(	self,
		fileNameIn
	)

Definition at line 287 of file runTheMatrix.py.

00288                                     :
00289         
00290         prefix = self.filesPrefMap[fileNameIn]
00291 
00292         print "processing ", fileNameIn,
00293         lines = []
00294         try:
00295             try:
00296                 inFile = open(fileNameIn, 'r')
00297                 print ' from local developer area',
00298             except IOError:
00299                 baseRelPath = os.environ['CMSSW_BASE']
00300                 # print "trying fall-back to cmsDriver files from developer area at:", baseRelPath
00301                 try:
00302                     inFile = open( os.path.join(baseRelPath, 'src/Configuration/PyReleaseValidation/data' ,fileNameIn), 'r')
00303                     print ' from ', baseRelPath,
00304                 except IOError:
00305                     baseRelPath = os.environ['CMSSW_RELEASE_BASE']
00306                     # print "trying fall back to cmsDriver files from base release at:", baseRelPath
00307                     inFile = open( os.path.join(baseRelPath, 'src/Configuration/PyReleaseValidation/data' ,fileNameIn), 'r')
00308                     print ' from ', baseRelPath,
00309             lines = inFile.readlines()
00310             inFile.close()
00311         except Exception, e:
00312             print "ERROR reading in file ", fileNameIn, str(e)
00313             return
00314 
00315         print " found ", len(lines), 'entries.'
00316         
00317         realRe = re.compile('\s*([1-9][0-9]*\.*[0-9]*)\s*\+\+\s*(.*?)\s*\+\+\s*(.*?)\s*\+\+\s*(.*?)\s*@@@\s*(.*)\s*')
00318         step1Re = re.compile('\s*([1-9][0-9]*\.*[0-9]*)\s*\+\+\s*(.*?)\s*\+\+\s*(.*?)\s*@@@\s*(.*)\s*')
00319         step2Re = re.compile('\s*STEP2\s*\+\+\s*(\S*)\s*@@@\s*(.*)\s*')
00320         step3Re = re.compile('\s*STEP3\s*\+\+\s*(\S*)\s*@@@\s*(.*)\s*')
00321         step4Re = re.compile('\s*STEP4\s*\+\+\s*(\S*)\s*@@@\s*(.*)\s*')
00322         for lineIn in lines:
00323             line = lineIn.strip()
00324 
00325             realMatch = realRe.match(line)
00326             if realMatch :
00327                 num  = realMatch.group(1).strip()
00328                 name = realMatch.group(2).strip().replace('<','').replace('>','').replace(':','')
00329                 next = realMatch.group(3).strip().replace('+','').replace(',', ' ')
00330                 cmd  = realMatch.group(4).strip()
00331 
00332                 step2 = "None"
00333                 step3 = "None"
00334                 step4 = "None"
00335 
00336                 steps = next.split()
00337                 if len(steps) > 0:
00338                     step2 = steps[0].strip()
00339                 if len(steps) > 1:
00340                     step3 = steps[1].strip()
00341                 if len(steps) > 2:
00342                     step4 = steps[2].strip()
00343                 
00344                 self.step1WorkFlows[(float(num),prefix)] = (str(float(num)), name, step2, step3, step4, cmd, None)
00345                 continue
00346             
00347                 
00348             step1Match = step1Re.match(line)
00349             if step1Match :
00350                 num  = step1Match.group(1).strip()
00351                 name = step1Match.group(2).strip().replace('<','').replace('>','').replace(':','')
00352                 next = step1Match.group(3).strip().replace('+','').replace(',', ' ')
00353                 cmd  = step1Match.group(4).strip()
00354                 step2 = "None"
00355                 step3 = "None"
00356                 step4 = "None"
00357 
00358                 steps = next.split()
00359                 if len(steps) > 0:
00360                     step2 = steps[0].strip()
00361                 if len(steps) > 1:
00362                     step3 = steps[1].strip()
00363                 if len(steps) > 2:
00364                     step4 = steps[2].strip()
00365                 
00366                 self.step1WorkFlows[(float(num),prefix)] = (str(float(num)), name, step2, step3, step4, cmd, None)
00367                 continue
00368             
00369             step2Match = step2Re.match(line)
00370             if step2Match :
00371                 name = step2Match.group(1).strip()
00372                 cmd  = step2Match.group(2).strip()
00373                 self.step2WorkFlows[name] = (cmd.replace('--no_exec','') ) # make sure the command is really run
00374                 continue
00375 
00376             step3Match = step3Re.match(line)
00377             if step3Match :
00378                 name = step3Match.group(1).strip()
00379                 cmd  = step3Match.group(2).strip()
00380                 self.step3WorkFlows[name] = ( cmd.replace('--no_exec','') ) # make sure the command is really run
00381                 continue
00382 
00383             step4Match = step4Re.match(line)
00384             if step4Match :
00385                 name = step4Match.group(1).strip()
00386                 cmd  = step4Match.group(2).strip()
00387                 self.step4WorkFlows[name] = ( cmd.replace('--no_exec','') ) # make sure the command is really run
00388                 continue
00389 
00390         return

def runTheMatrix::MatrixReader::reset ( self )

Definition at line 260 of file runTheMatrix.py.

00261                    :
00262 
00263         self.step1WorkFlows = {}
00264         self.step2WorkFlows = {}
00265         self.step3WorkFlows = {}
00266         self.step4WorkFlows = {}
00267 
00268         self.workFlows = []
00269         self.nameList  = {}
00270         
00271 
00272         self.filesPrefMap = {'cmsDriver_standard_hlt.txt' : 'std-' ,
00273                              'cmsDriver_highstats_hlt.txt': 'hi-'  ,
00274                              'cmsDriver_generator.txt'    : 'gen-'  ,
00275                              # 'cmsDriver_PileUp_hlt.txt'   : 'pu-'  ,
00276                              # 'cmsDriver_realData_hlt.txt' : 'data-'
00277                              }
00278 
00279         self.files = ['cmsDriver_standard_hlt.txt' ,
00280                       'cmsDriver_highstats_hlt.txt',
00281                       'cmsDriver_generator.txt'    ,
00282                       # 'cmsDriver_PileUp_hlt.txt'   ,
00283                       # 'cmsDriver_realData_hlt.txt' 
00284                       ]
00285         
00286         return

def runTheMatrix::MatrixReader::show	(	self,
		selected = `None`
	)

Definition at line 517 of file runTheMatrix.py.

00517                                  :    
00518         # self.showRaw()
00519         self.showWorkFlows(selected)
00520         print '\n','-'*80,'\n'
00521 
00522

def runTheMatrix::MatrixReader::showRaw ( self )

Definition at line 391 of file runTheMatrix.py.

00392                      :
00393 
00394         print "found ", len(self.step1WorkFlows.keys()), ' workflows for step1:'
00395         ids = self.step1WorkFlows.keys()
00396         ids.sort()
00397         for key in ids:
00398             val = self.step1WorkFlows[key]
00399             print key[0], ':', val
00400         
00401         print "found ", len(self.step2WorkFlows.keys()), ' workflows for step2:'
00402         for key, val in self.step2WorkFlows.items():
00403             print key, ':', val
00404         
00405         print "found ", len(self.step3WorkFlows.keys()), ' workflows for step3:'
00406         for key, val in self.step3WorkFlows.items():
00407             print key, ':', val
00408         
00409         print "found ", len(self.step4WorkFlows.keys()), ' workflows for step4:'
00410         for key, val in self.step4WorkFlows.items():
00411             print key, ':', val
00412         
00413         return

def runTheMatrix::MatrixReader::showWorkFlows	(	self,
		selected = `None`
	)

Definition at line 414 of file runTheMatrix.py.

00415                                           :
00416 
00417         maxLen = 100 # for summary, limit width of output
00418         fmt1   = "%-6s %-35s [1]: %s ..."
00419         fmt2   = "       %35s [%d]: %s ..."
00420         print "\nfound a total of ", len(self.workFlows), ' workflows:'
00421         if selected:
00422             print "      of which the following", len(selected), 'were selected:'
00423             maxLen = -1  # for individual listing, no limit on width
00424             fmt1   = "%-6s %-35s [1]: %s " 
00425             fmt2   = "       %35s [%d]: %s"
00426         n1 = 0
00427         n2 = 0
00428         n3 = 0
00429         n4 = 0
00430         for wf in self.workFlows:
00431             if selected and float(wf.numId) not in selected: continue
00432             n1+=1
00433             print fmt1 % (wf.numId, wf.nameId, (wf.cmdStep1+' ')[:maxLen])
00434             if wf.cmdStep2:
00435                 n2+=1
00436                 print fmt2 % ( ' ', 2, (wf.cmdStep2+' ')[:maxLen])
00437                 if wf.cmdStep3:
00438                     n3+=1
00439                     print fmt2 % ( ' ', 3, (wf.cmdStep3+' ')[:maxLen])
00440                     if wf.cmdStep4:
00441                         n4+=1
00442                         print fmt2 % ( ' ', 4, (wf.cmdStep4+' ')[:maxLen])
00443 
00444         print n1, 'workflows for step1,'
00445         print n2, 'workflows for step1 + step2,'
00446         print n3, 'workflows for step1 + step2 + step3'
00447         print n4, 'workflows for step1 + step2 + step3 + step4'
00448 
00449         return

def runTheMatrix::MatrixReader::updateDB ( self )

Definition at line 523 of file runTheMatrix.py.

00524                       :
00525 
00526         import pickle
00527         pickle.dump(self.workFlows, open('theMatrix.pkl', 'w') )
00528 
00529         return
00530 
00531 # ================================================================================