CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes
runTheMatrix_dev.MatrixReader Class Reference
Inheritance diagram for runTheMatrix_dev.MatrixReader:

Public Member Functions

def __init__
 
def createWorkFlows
 
def makeCmd
 
def prepare
 
def readMatrix
 
def reset
 
def show
 
def showRaw
 
def showWorkFlows
 
def updateDB
 

Public Attributes

 files
 
 filesPrefMap
 
 nameList
 
 relvalModule
 
 step1WorkFlows
 
 step2WorkFlows
 
 step3WorkFlows
 
 step4WorkFlows
 
 workFlows
 

Detailed Description

Definition at line 254 of file runTheMatrix_dev.py.

Constructor & Destructor Documentation

def runTheMatrix_dev.MatrixReader.__init__ (   self)

Definition at line 256 of file runTheMatrix_dev.py.

References TPedResult.reset(), BinningPointByMap.reset(), ora::NamedReference.reset(), AlcaBeamSpotManager.reset(), ora::IRelationalOperation.reset(), ApvLatencyAnalysis.reset(), ora::IteratorBuffer.reset(), ora::Handle< T >.reset(), TB06Tree.reset(), ora::ContainerIterator.reset(), VEcalCalibBlock.reset(), EcalCondHeader.reset(), CondIter< DataT >.reset(), SamplingAnalysis.reset(), IMACalibBlock.reset(), DaqScopeModeAnalysis.reset(), L3CalibBlock.reset(), ora::MultiRecordInsertOperation.reset(), pos::PixelROCStatus.reset(), FedTimingAnalysis.reset(), L1MuDTTFMasks.reset(), L1MuDTTFParameters.reset(), TB06TreeH2.reset(), L1MuDTPhiLut.reset(), L1MuDTPtaLut.reset(), cond::BaseValueExtractor< T >.reset(), L1MuDTEtaPatternLut.reset(), FedCablingAnalysis.reset(), L1MuDTQualPatternLut.reset(), PhysicsTools::Calibration::Histogram< Value_t, Axis_t >.reset(), PedsOnlyAnalysis.reset(), L1MuDTExtLut.reset(), PhysicsTools::Calibration::Histogram2D< Value_t, AxisX_t, AxisY_t >.reset(), VpspScanAnalysis.reset(), TB06Reco.reset(), NoiseAnalysis.reset(), PedestalsAnalysis.reset(), OptoScanAnalysis.reset(), PhysicsTools::Calibration::Histogram3D< Value_t, AxisX_t, AxisY_t, AxisZ_t >.reset(), TrackerMap.reset(), ApvTimingAnalysis.reset(), CalibrationAnalysis.reset(), FastFedCablingAnalysis.reset(), ora::UniqueRef< T >.reset(), PedsFullNoiseAnalysis.reset(), CommissioningAnalysis.reset(), ora::InsertOperation.reset(), ora::Ptr< T >.reset(), TB06RecoH2.reset(), ora::BulkInsertOperation.reset(), cond::XMLAuthenticationService::XMLAuthenticationService.reset(), ora::UpdateOperation.reset(), ora::DeleteOperation.reset(), ora::QueryableVector< Tp >.reset(), DTTFBitArray< N >.reset(), runTheMatrix.MatrixReader.reset(), runTheMatrix_dev.MatrixReader.reset(), and ora::LoaderClient.reset().

257  def __init__(self):
258 
259  self.reset()
260 
261  return

Member Function Documentation

def runTheMatrix_dev.MatrixReader.createWorkFlows (   self,
  fileNameIn 
)

Definition at line 466 of file runTheMatrix_dev.py.

References runTheMatrix.MatrixReader.filesPrefMap, runTheMatrix_dev.MatrixReader.filesPrefMap, runTheMatrix.MatrixReader.nameList, runTheMatrix_dev.MatrixReader.nameList, runTheMatrix.MatrixReader.step1WorkFlows, and runTheMatrix_dev.MatrixReader.step1WorkFlows.

Referenced by runTheMatrix_dev.MatrixReader.prepare().

467  def createWorkFlows(self, fileNameIn):
468 
469  prefixIn = self.filesPrefMap[fileNameIn]
470 
471  # get through the list of items and update the requested workflows only
472  keyList = self.step1WorkFlows.keys()
473  ids = []
474  for item in keyList:
475  id, pref = item
476  if pref != prefixIn : continue
477  ids.append( float(id) )
478 
479  ids.sort()
480  n1 = 0
481  n2 = 0
482  n3 = 0
483  n4 = 0
484  for key in ids:
485  val = self.step1WorkFlows[(key,prefixIn)]
486  num, name, cmd, step2, step3, step4, inputInfo = val
487  nameId = num+'_'+name
488  if nameId in self.nameList.keys():
489  print "==> duplicate name found for ", nameId
490  print ' keeping : ', self.nameList[nameId]
491  print ' ignoring : ', val
492  else:
493  self.nameList[nameId] = val
494 
495  cmd2 = None
496  cmd3 = None
497  cmd4 = None
498 
499  n1 += 1
500 
501  if step2.lower() != '':
502  n2 += 1
503  cmd2 = step2
504  if step3.lower() != '':
505  n3 += 1
506  cmd3 = step3
507  if step4.lower() != '':
508  n4 += 1
509  cmd4 = step4
510  #print '\tstep3 : ', self.step3WorkFlows[step3]
511  self.workFlows.append( WorkFlow(num, name, cmd, cmd2, cmd3, cmd4, inputInfo) )
512 
513  return
def runTheMatrix_dev.MatrixReader.makeCmd (   self,
  step 
)

Definition at line 286 of file runTheMatrix_dev.py.

Referenced by runTheMatrix_dev.MatrixReader.readMatrix(), and runTheMatrix_dev.MatrixReader.showRaw().

287  def makeCmd(self, step):
288 
289  cmd = ''
290  cfg = None
291  input = None
292  #print step
293  #print defaults
294  for k,v in step.items():
295  if 'no_exec' in k : continue # we want to really run it ...
296  if k.lower() == 'cfg':
297  cfg = v
298  continue # do not append to cmd, return separately
299  if k.lower() == 'input':
300  input = v
301  continue # do not append to cmd, return separately
302  #print k,v
303  cmd += ' ' + k + ' ' + str(v)
304  return cfg, input, cmd
def runTheMatrix_dev.MatrixReader.prepare (   self,
  useInput = None 
)

Definition at line 514 of file runTheMatrix_dev.py.

References runTheMatrix.MatrixReader.createWorkFlows(), runTheMatrix_dev.MatrixReader.createWorkFlows(), relval_steps.InputInfo.files, runTheMatrix_dev.MatrixReader.files, runTheMatrix.MatrixReader.files, runTheMatrix.MatrixReader.readMatrix(), and runTheMatrix_dev.MatrixReader.readMatrix().

Referenced by addOnTests.StandardTester.runTests().

515  def prepare(self, useInput=None):
516 
517  for matrixFile in self.files:
518  try:
519  self.readMatrix(matrixFile, useInput)
520  except Exception, e:
521  print "ERROR reading file:", matrixFile, str(e)
522  raise
523 
524  try:
525  self.createWorkFlows(matrixFile)
526  except Exception, e:
527  print "ERROR creating workflows :", str(e)
528  raise
def runTheMatrix_dev.MatrixReader.readMatrix (   self,
  fileNameIn,
  useInput = None 
)

Definition at line 305 of file runTheMatrix_dev.py.

References runTheMatrix.MatrixReader.filesPrefMap, runTheMatrix_dev.MatrixReader.filesPrefMap, runTheMatrix_dev.MatrixReader.makeCmd(), runTheMatrix_dev.MatrixReader.relvalModule, runTheMatrix.MatrixReader.step1WorkFlows, and runTheMatrix_dev.MatrixReader.step1WorkFlows.

Referenced by runTheMatrix_dev.MatrixReader.prepare(), and runTheMatrix_dev.MatrixReader.showRaw().

306  def readMatrix(self, fileNameIn, useInput=None):
307 
308  prefix = self.filesPrefMap[fileNameIn]
309 
310  print "processing ", fileNameIn
311 
312  try:
313  _tmpMod = __import__( 'Configuration.PyReleaseValidation.'+fileNameIn )
314  self.relvalModule = sys.modules['Configuration.PyReleaseValidation.'+fileNameIn]
315  except Exception, e:
316  print "ERROR importing file ", fileNameIn, str(e)
317  return
318 
319  print "request for INPUT for ", useInput
320 
321  for num, wfInfo in self.relvalModule.workflows.items():
322  wfName = wfInfo[0]
323  stepList = wfInfo[1]
324  # if no explicit name given for the workflow, use the name of step1
325  if wfName.strip() == '': wfName = stepList[0]
326  stepCmds = ['','','','']
327  stepIndex = 0
328  name = wfName
329  inputInfo = None
330  for step in stepList:
331  if len(name) > 0 : name += '+'
332  stepName = step
333  if stepIndex==0 and useInput and (str(num) in useInput or "all" in useInput):
334  # print "--> using INPUT as step1 for workflow ", num
335  if step+'INPUT' in self.relvalModule.step1.keys():
336  stepName = step+"INPUT"
337  name += stepName
338  cfg, input, opts = self.makeCmd(self.relvalModule.stepList[stepIndex][stepName])
339  if input and cfg :
340  msg = "FATAL ERROR: found both cfg and input for workflow "+str(num)+' step '+stepName
341  raise msg
342 
343  if cfg:
344  cmd = 'cmsDriver.py '+cfg+' '+opts
345  if stepIndex==0 and not inputInfo and input: # only if we didn't already set the input
346  inputInfo = input
347  cmd = 'DATAINPUT from '+inputInfo.dataSet
348 
349  if stepIndex > 0:
350  cmd = 'cmsDriver.py step'+str(stepIndex+1)+'.py '+opts
351 
352  stepCmds[stepIndex] = cmd
353  stepIndex += 1
354 
355  self.step1WorkFlows[(float(num),prefix)] = (str(float(num)), name, stepCmds[0], stepCmds[1], stepCmds[2], stepCmds[3], inputInfo)
356 
357  return
def runTheMatrix_dev.MatrixReader.reset (   self)

Definition at line 262 of file runTheMatrix_dev.py.

Referenced by runTheMatrix_dev.MatrixReader.__init__(), and runTheMatrix_dev.MatrixReader.showRaw().

263  def reset(self):
265  self.step1WorkFlows = {}
266  self.step2WorkFlows = {}
267  self.step3WorkFlows = {}
268  self.step4WorkFlows = {}
270  self.workFlows = []
271  self.nameList = {}
272 
273  self.filesPrefMap = {'relval_standard' : 'std-' ,
274  'relval_highstats': 'hi-' ,
275  'relval_generator': 'gen-' ,
276  }
278  self.files = ['relval_standard' ,
279  'relval_highstats',
280  'relval_generator',
281  ]
283  self.relvalModule = None
284 
285  return
def runTheMatrix_dev.MatrixReader.show (   self,
  selected = None 
)

Definition at line 529 of file runTheMatrix_dev.py.

References runTheMatrix.MatrixReader.showWorkFlows(), and runTheMatrix_dev.MatrixReader.showWorkFlows().

Referenced by python.Vispa.Main.AboutDialog.AboutDialog.onScreen(), python.Vispa.Gui.BoxContentDialog.BoxContentDialog.onScreen(), and python.Vispa.Gui.FindDialog.FindDialog.onScreen().

530  def show(self, selected=None):
531  # self.showRaw()
532  self.showWorkFlows(selected)
533  print '\n','-'*80,'\n'
534 
def runTheMatrix_dev.MatrixReader.showRaw (   self,
  useInput 
)

Definition at line 358 of file runTheMatrix_dev.py.

References relval_steps.InputInfo.files, runTheMatrix_dev.MatrixReader.files, runTheMatrix.MatrixReader.files, runTheMatrix_dev.MatrixReader.makeCmd(), runTheMatrix.MatrixReader.readMatrix(), runTheMatrix_dev.MatrixReader.readMatrix(), TPedResult.reset(), BinningPointByMap.reset(), ora::NamedReference.reset(), AlcaBeamSpotManager.reset(), ora::IRelationalOperation.reset(), ApvLatencyAnalysis.reset(), ora::Handle< T >.reset(), ora::IteratorBuffer.reset(), TB06Tree.reset(), ora::ContainerIterator.reset(), VEcalCalibBlock.reset(), EcalCondHeader.reset(), CondIter< DataT >.reset(), SamplingAnalysis.reset(), IMACalibBlock.reset(), DaqScopeModeAnalysis.reset(), ora::MultiRecordInsertOperation.reset(), L3CalibBlock.reset(), FedTimingAnalysis.reset(), pos::PixelROCStatus.reset(), L1MuDTTFMasks.reset(), L1MuDTTFParameters.reset(), TB06TreeH2.reset(), L1MuDTPhiLut.reset(), L1MuDTPtaLut.reset(), cond::BaseValueExtractor< T >.reset(), L1MuDTEtaPatternLut.reset(), FedCablingAnalysis.reset(), L1MuDTQualPatternLut.reset(), PhysicsTools::Calibration::Histogram< Value_t, Axis_t >.reset(), PedsOnlyAnalysis.reset(), L1MuDTExtLut.reset(), PhysicsTools::Calibration::Histogram2D< Value_t, AxisX_t, AxisY_t >.reset(), VpspScanAnalysis.reset(), TB06Reco.reset(), NoiseAnalysis.reset(), PedestalsAnalysis.reset(), OptoScanAnalysis.reset(), PhysicsTools::Calibration::Histogram3D< Value_t, AxisX_t, AxisY_t, AxisZ_t >.reset(), TrackerMap.reset(), ApvTimingAnalysis.reset(), CalibrationAnalysis.reset(), FastFedCablingAnalysis.reset(), ora::UniqueRef< T >.reset(), PedsFullNoiseAnalysis.reset(), CommissioningAnalysis.reset(), ora::InsertOperation.reset(), ora::Ptr< T >.reset(), TB06RecoH2.reset(), ora::BulkInsertOperation.reset(), cond::XMLAuthenticationService::XMLAuthenticationService.reset(), ora::UpdateOperation.reset(), ora::DeleteOperation.reset(), ora::QueryableVector< Tp >.reset(), DTTFBitArray< N >.reset(), runTheMatrix.MatrixReader.reset(), runTheMatrix_dev.MatrixReader.reset(), ora::LoaderClient.reset(), runTheMatrix.MatrixReader.step1WorkFlows, and runTheMatrix_dev.MatrixReader.step1WorkFlows.

359  def showRaw(self, useInput):
360 
361  for matrixFile in self.files:
362  self.reset()
363  try:
364  self.readMatrix(matrixFile, useInput)
365  except Exception, e:
366  print "ERROR reading file:", matrixFile, str(e)
367  raise
368 
369  if not self.step1WorkFlows: continue
370 
371  dataFileName = matrixFile.replace('relval_', 'cmsDriver_')+'_hlt.txt'
372  outFile = open(dataFileName,'w')
373 
374  print "found ", len(self.step1WorkFlows.keys()), ' workflows for ', dataFileName
375  ids = self.step1WorkFlows.keys()
376  ids.sort()
377  stepCmds = ['','','','']
378  for key in ids:
379  num, name, stepCmds[0], stepCmds[1], stepCmds[2], stepCmds[3], inputInfo = self.step1WorkFlows[key]
380  wfName,stepNames= name.split('+',1)
381  otherSteps = None
382  if '+' in stepNames:
383  step1,otherSteps = stepNames.split('+',1)
384  line = num + ' ++ '+ wfName
385  if otherSteps:
386  line += ' ++ ' +otherSteps.replace('+',',')
387  else:
388  line += ' ++ none'
389  if inputInfo :
390  line += ' ++ REALDATA: '+inputInfo.dataSet
391  line += ', FILES: ' +str(inputInfo.files)
392  line += ', EVENTS: '+str(inputInfo.events)
393  line += ', LABEL: ' +inputInfo.label
394  line += ', LOCATION:'+inputInfo.location
395  line += ' @@@'
396  else:
397  line += ' @@@ '+stepCmds[0]
398  print line
399  outFile.write(line+'\n')
400 
401  outFile.write('\n'+'\n')
402  for stepName in self.relvalModule.step2.keys():
403  cfg,input,cmd = self.makeCmd(self.relvalModule.step2[stepName])
404  line = 'STEP2 ++ ' +stepName + ' @@@ cmsDriver.py step2 ' +cmd
405  print line
406  outFile.write(line+'\n')
407 
408  outFile.write('\n'+'\n')
409  for stepName in self.relvalModule.step3.keys():
410  cfg,input,cmd = self.makeCmd(self.relvalModule.step3[stepName])
411  line ='STEP3 ++ ' +stepName + ' @@@ cmsDriver.py step3_RELVAL ' +cmd
412  print line
413  outFile.write(line+'\n')
414 
415  outFile.write('\n'+'\n')
416  for stepName in self.relvalModule.step4.keys():
417  cfg,input,cmd = self.makeCmd(self.relvalModule.step4[stepName])
418  line = 'STEP4 ++ ' +stepName + ' @@@ cmsDriver.py step4 ' +cmd
419  print line
420  outFile.write(line+'\n')
421 
422  outFile.close()
423 
424 
425  return
def runTheMatrix_dev.MatrixReader.showWorkFlows (   self,
  selected = None 
)

Definition at line 426 of file runTheMatrix_dev.py.

References runTheMatrix.MatrixReader.workFlows, runTheMatrix_dev.MatrixReader.workFlows, and runTheMatrix.MatrixRunner.workFlows.

Referenced by runTheMatrix_dev.MatrixReader.show().

427  def showWorkFlows(self, selected=None):
428 
429  maxLen = 100 # for summary, limit width of output
430  fmt1 = "%-6s %-35s [1]: %s ..."
431  fmt2 = " %35s [%d]: %s ..."
432  print "\nfound a total of ", len(self.workFlows), ' workflows:'
433  if selected:
434  print " of which the following", len(selected), 'were selected:'
435  #-ap for now:
436  maxLen = -1 # for individual listing, no limit on width
437  fmt1 = "%-6s %-35s [1]: %s "
438  fmt2 = " %35s [%d]: %s"
439 
440  n1 = 0
441  n2 = 0
442  n3 = 0
443  n4 = 0
444  for wf in self.workFlows:
445  if selected and float(wf.numId) not in selected: continue
446  print ''
447  n1+=1
448  wfName, stepNames = wf.nameId.split('+',1)
449  print fmt1 % (wf.numId, stepNames, (wf.cmdStep1+' ')[:maxLen])
450  if wf.cmdStep2:
451  n2+=1
452  print fmt2 % ( ' ', 2, (wf.cmdStep2+' ')[:maxLen])
453  if wf.cmdStep3:
454  n3+=1
455  print fmt2 % ( ' ', 3, (wf.cmdStep3+' ')[:maxLen])
456  if wf.cmdStep4:
457  n4+=1
458  print fmt2 % ( ' ', 4, (wf.cmdStep4+' ')[:maxLen])
459 
460  print n1, 'workflows for step1,'
461  print n2, 'workflows for step1 + step2,'
462  print n3, 'workflows for step1 + step2 + step3'
463  print n4, 'workflows for step1 + step2 + step3 + step4'
464 
465  return
def runTheMatrix_dev.MatrixReader.updateDB (   self)

Definition at line 535 of file runTheMatrix_dev.py.

References runTheMatrix.MatrixReader.workFlows, runTheMatrix_dev.MatrixReader.workFlows, and runTheMatrix.MatrixRunner.workFlows.

536  def updateDB(self):
537 
538  import pickle
539  pickle.dump(self.workFlows, open('theMatrix.pkl', 'w') )
540 
541  return
542 
543 # ================================================================================

Member Data Documentation

runTheMatrix_dev.MatrixReader.files

Definition at line 277 of file runTheMatrix_dev.py.

Referenced by runTheMatrix_dev.MatrixReader.prepare(), and runTheMatrix_dev.MatrixReader.showRaw().

runTheMatrix_dev.MatrixReader.filesPrefMap

Definition at line 272 of file runTheMatrix_dev.py.

Referenced by runTheMatrix_dev.MatrixReader.createWorkFlows(), and runTheMatrix_dev.MatrixReader.readMatrix().

runTheMatrix_dev.MatrixReader.nameList

Definition at line 270 of file runTheMatrix_dev.py.

Referenced by runTheMatrix_dev.MatrixReader.createWorkFlows().

runTheMatrix_dev.MatrixReader.relvalModule

Definition at line 282 of file runTheMatrix_dev.py.

Referenced by runTheMatrix_dev.MatrixReader.readMatrix().

runTheMatrix_dev.MatrixReader.step1WorkFlows

Definition at line 264 of file runTheMatrix_dev.py.

Referenced by runTheMatrix_dev.MatrixReader.createWorkFlows(), runTheMatrix_dev.MatrixReader.readMatrix(), and runTheMatrix_dev.MatrixReader.showRaw().

runTheMatrix_dev.MatrixReader.step2WorkFlows

Definition at line 265 of file runTheMatrix_dev.py.

runTheMatrix_dev.MatrixReader.step3WorkFlows

Definition at line 266 of file runTheMatrix_dev.py.

runTheMatrix_dev.MatrixReader.step4WorkFlows

Definition at line 267 of file runTheMatrix_dev.py.

runTheMatrix_dev.MatrixReader.workFlows

Definition at line 269 of file runTheMatrix_dev.py.

Referenced by runTheMatrix_dev.MatrixRunner.runTests(), runTheMatrix_dev.MatrixReader.showWorkFlows(), and runTheMatrix_dev.MatrixReader.updateDB().