test
CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Private Member Functions
looper.Looper Class Reference
Inheritance diagram for looper.Looper:

Public Member Functions

def __init__
 
def loop
 
def process
 
def write
 

Public Attributes

 analyzers
 
 cfg_comp
 
 classes
 
 config
 
 event
 
 events
 
 firstEvent
 
 iEvent
 
 logger
 
 memLast
 
 memReportFirstEvent
 
 name
 
 nEvents
 
 nPrint
 
 outDir
 
 setup
 
 start_time
 
 start_time_event
 
 timeReport
 

Private Member Functions

def _build
 
def _prepareOutput
 

Detailed Description

Creates a set of analyzers, and schedules the event processing.

Definition at line 49 of file looper.py.

Constructor & Destructor Documentation

def looper.Looper.__init__ (   self,
  name,
  config,
  nEvents = None,
  firstEvent = 0,
  nPrint = 0,
  timeReport = False,
  quiet = False,
  memCheckFromEvent = -1 
)
Handles the processing of an event sample.
An Analyzer is built for each Config.Analyzer present
in sequence. The Looper can then be used to process an event,
or a collection of events.

Parameters:
name    : name of the Looper, will be used as the output directory name
config  : process configuration information, see Config
nEvents : number of events to process. Defaults to all.
firstEvent : first event to process. Defaults to the first one.
nPrint  : number of events to print at the beginning

Definition at line 59 of file looper.py.

59 
60  memCheckFromEvent=-1):
61  """Handles the processing of an event sample.
62  An Analyzer is built for each Config.Analyzer present
63  in sequence. The Looper can then be used to process an event,
64  or a collection of events.
65 
66  Parameters:
67  name : name of the Looper, will be used as the output directory name
68  config : process configuration information, see Config
69  nEvents : number of events to process. Defaults to all.
70  firstEvent : first event to process. Defaults to the first one.
71  nPrint : number of events to print at the beginning
72  """
73 
74  self.config = config
75  self.name = self._prepareOutput(name)
76  self.outDir = self.name
77  self.logger = logging.getLogger( self.name )
78  self.logger.addHandler(logging.FileHandler('/'.join([self.name,
79  'log.txt'])))
80  self.logger.propagate = False
81  if not quiet:
82  self.logger.addHandler( logging.StreamHandler(sys.stdout) )
83 
84  self.cfg_comp = config.components[0]
85  self.classes = {}
86  self.analyzers = map( self._build, config.sequence )
87  self.nEvents = nEvents
88  self.firstEvent = firstEvent
89  self.nPrint = int(nPrint)
90  self.timeReport = [ {'time':0.0,'events':0} for a in self.analyzers ] if timeReport else False
91  self.memReportFirstEvent = memCheckFromEvent
92  self.memLast=0
93  tree_name = None
94  if( hasattr(self.cfg_comp, 'tree_name') ):
95  tree_name = self.cfg_comp.tree_name
96  if len(self.cfg_comp.files)==0:
97  errmsg = 'please provide at least an input file in the files attribute of this component\n' + str(self.cfg_comp)
98  raise ValueError( errmsg )
99  if hasattr(config,"preprocessor") and config.preprocessor is not None :
100  self.cfg_comp = config.preprocessor.run(self.cfg_comp,self.outDir,firstEvent,nEvents)
101  if hasattr(self.cfg_comp,"options"):
102  print self.cfg_comp.files,self.cfg_comp.options
103  self.events = config.events_class(self.cfg_comp.files, tree_name,options=self.cfg_comp.options)
104  else :
105  self.events = config.events_class(self.cfg_comp.files, tree_name)
106  if hasattr(self.cfg_comp, 'fineSplit'):
107  fineSplitIndex, fineSplitFactor = self.cfg_comp.fineSplit
108  if fineSplitFactor > 1:
109  if len(self.cfg_comp.files) != 1:
110  raise RuntimeError("Any component with fineSplit > 1 is supposed to have just a single file, while %s has %s" % (self.cfg_comp.name, self.cfg_comp.files))
111  totevents = min(len(self.events),int(nEvents)) if (nEvents and int(nEvents) not in [-1,0]) else len(self.events)
112  self.nEvents = int(ceil(totevents/float(fineSplitFactor)))
113  self.firstEvent = firstEvent + fineSplitIndex * self.nEvents
114  if self.firstEvent + self.nEvents >= totevents:
115  self.nEvents = totevents - self.firstEvent
116  #print "For component %s will process %d events starting from the %d one, ending at %d excluded" % (self.cfg_comp.name, self.nEvents, self.firstEvent, self.nEvents + self.firstEvent)
117  # self.event is set in self.process
118  self.event = None
119  services = dict()
120  for cfg_serv in config.services:
121  service = self._build(cfg_serv)
122  services[cfg_serv.name] = service
123  # would like to provide a copy of the config to the setup,
124  # so that analyzers cannot modify the config of other analyzers.
125  # but cannot copy the autofill config.
126  self.setup = Setup(config, services)
def _prepareOutput
Definition: looper.py:132
memReportFirstEvent
Definition: looper.py:90
def _build
Definition: looper.py:127
T min(T a, T b)
Definition: MathUtil.h:58
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
if(dp >Float(M_PI)) dp-

Member Function Documentation

def looper.Looper._build (   self,
  cfg 
)
private

Definition at line 127 of file looper.py.

References analyzer.Analyzer.cfg_comp, looper.Looper.cfg_comp, and looper.Looper.outDir.

128  def _build(self, cfg):
129  theClass = cfg.class_object
130  obj = theClass( cfg, self.cfg_comp, self.outDir )
131  return obj
def _build
Definition: looper.py:127
def looper.Looper._prepareOutput (   self,
  name 
)
private

Definition at line 132 of file looper.py.

133  def _prepareOutput(self, name):
134  index = 0
135  tmpname = name
136  while True and index < 2000:
137  try:
138  # print 'mkdir', self.name
139  os.mkdir( tmpname )
140  break
141  except OSError:
142  index += 1
143  tmpname = '%s_%d' % (name, index)
144  if index == 2000:
145  raise ValueError( "More than 2000 output folder with same name or 2000 attempts failed, please clean-up, change name or check permissions")
146  return tmpname
147 
def _prepareOutput
Definition: looper.py:132
def looper.Looper.loop (   self)
Loop on a given number of events.

At the beginning of the loop, 
Analyzer.beginLoop is called for each Analyzer.
At each event, self.process is called.
At the end of the loop, Analyzer.endLoop is called.

Definition at line 148 of file looper.py.

References looper.Looper.analyzers, Config.Process.analyzers, analyzer.Analyzer.cfg_comp, looper.Looper.cfg_comp, generateEDF.LumiInfo.events, Forest.events, HitEff.events, Node.events, ME::Header.events, looper.Looper.events, MatrixUtil.InputInfo.events, CosmicRateAnalyzer.events, options.HLTProcessOptions.events, JsonOutputProducer::JsonConfiguration.events, Printer.Printer.firstEvent, lhef::LHEReader.firstEvent, StudyHLT.firstEvent, ApeEstimatorSummary.firstEvent, looper.Looper.firstEvent, LeptonRecoSkim.firstEvent, SiStripMonitorPedestals.firstEvent, IsoTrackCalib.firstEvent, SiStripMonitorDigi.firstEvent, SiStripMonitorCluster.firstEvent, SiStripMonitorTrack.firstEvent, HiggsTo2GammaSkim.nEvents, HiggsToZZ4LeptonsSkim.nEvents, JGJFilter.nEvents, HiggsToZZ4LeptonsSkimEff.nEvents, MCDijetResonance.nEvents, EgammaProbeSelector.nEvents, QualityTester.nEvents, AlCaHcalNoiseProducer.nEvents, DTScalerInfoTask.nEvents, MCatNLOSource.nEvents, AlpgenHeader.nEvents, HeavyChHiggsToTauNuSkim.nEvents, DTLocalTriggerLutTask.nEvents, looper.Looper.nEvents, DTLocalTriggerBaseTask.nEvents, MuonIsolationDQM.nEvents, MatacqProducer::stats_t.nEvents, cscdqm::Configuration.nEvents, event.Event.setup, and looper.Looper.setup.

149  def loop(self):
150  """Loop on a given number of events.
151 
152  At the beginning of the loop,
153  Analyzer.beginLoop is called for each Analyzer.
154  At each event, self.process is called.
155  At the end of the loop, Analyzer.endLoop is called.
156  """
157  nEvents = self.nEvents
158  firstEvent = self.firstEvent
159  iEv = firstEvent
160  if nEvents is None or int(nEvents) > len(self.events) :
161  nEvents = len(self.events)
162  else:
163  nEvents = int(nEvents)
164  eventSize = nEvents
165  self.logger.info(
166  'starting loop at event {firstEvent} '\
167  'to process {eventSize} events.'.format(firstEvent=firstEvent,
168  eventSize=eventSize))
169  self.logger.info( str( self.cfg_comp ) )
170  for analyzer in self.analyzers:
171  analyzer.beginLoop(self.setup)
172  try:
173  for iEv in range(firstEvent, firstEvent+eventSize):
174  # if iEv == nEvents:
175  # break
176  if iEv%100 ==0:
177  # print 'event', iEv
178  if not hasattr(self,'start_time'):
179  print 'event', iEv
180  self.start_time = timeit.default_timer()
181  self.start_time_event = iEv
182  else:
183  print 'event %d (%.1f ev/s)' % (iEv, (iEv-self.start_time_event)/float(timeit.default_timer() - self.start_time))
184 
185  self.process( iEv )
186  if iEv<self.nPrint:
187  print self.event
188 
189  except UserWarning:
190  print 'Stopped loop following a UserWarning exception'
191 
192  info = self.logger.info
193  warning = self.logger.warning
194  warning('number of events processed: {nEv}'.format(nEv=iEv+1))
195  warning('')
196  info( self.cfg_comp )
197  info('')
198  for analyzer in self.analyzers:
199  analyzer.endLoop(self.setup)
200  if self.timeReport:
201  allev = max([x['events'] for x in self.timeReport])
202  warning("\n ---- TimeReport (all times in ms; first evt is skipped) ---- ")
203  warning("%9s %9s %9s %9s %6s %s" % ("processed","all evts","time/proc", " time/all", " [%] ", "analyer"))
204  warning("%9s %9s %9s %9s %6s %s" % ("---------","--------","---------", "---------", " -----", "-------------"))
205  sumtime = sum(rep['time'] for rep in self.timeReport)
206  passev = self.timeReport[-1]['events']
207  for ana,rep in zip(self.analyzers,self.timeReport):
208  timePerProcEv = rep['time']/(rep['events']-1) if rep['events'] > 1 else 0
209  timePerAllEv = rep['time']/(allev-1) if allev > 1 else 0
210  fracAllEv = rep['time']/sumtime
211  warning( "%9d %9d %10.2f %10.2f %5.1f%% %s" % ( rep['events'], allev, 1000*timePerProcEv, 1000*timePerAllEv, 100.0*fracAllEv, ana.name))
212  totPerProcEv = sumtime/(passev-1) if passev > 1 else 0
213  totPerAllEv = sumtime/(allev-1) if allev > 1 else 0
214  warning("%9s %9s %9s %9s %s" % ("---------","--------","---------", "---------", "-------------"))
215  warning("%9d %9d %10.2f %10.2f %5.1f%% %s" % ( passev, allev, 1000*totPerProcEv, 1000*totPerAllEv, 100.0, "TOTAL"))
216  warning("")
217  if hasattr(self.events, 'endLoop'): self.events.endLoop()
218  if hasattr(self.config,"preprocessor") and self.config.preprocessor is not None:
219  if hasattr(self.config.preprocessor,"endLoop"):
220  self.config.preprocessor.endLoop(self.cfg_comp)
static const TGPicture * info(bool iBackgroundIsBlack)
OutputIterator zip(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp)
def process
Definition: looper.py:221
def looper.Looper.process (   self,
  iEv 
)
Run event processing for all analyzers in the sequence.

This function is called by self.loop,
but can also be called directly from
the python interpreter, to jump to a given event.

Definition at line 221 of file looper.py.

References TB06Reco.event, TB06RecoH2.event, core.AutoHandle.AutoHandle.event, ApeOverview.event, L1Analysis::L1AnalysisEventDataFormat.event, MuonPair.event, LHAupLesHouches.event, EcalPerEvtMatacqAnalyzer.event, WZInterestingEventSelector::event.event, EcalStatusAnalyzer.event, lhef::LHEProxy.event, MCatNLOSource.event, EcalMatacqAnalyzer.event, HitEff.event, L1TCaloLayer1RawToDigi.event, looper.Looper.event, EcalTestPulseAnalyzer.event, EcalABAnalyzer.event, EcalPerEvtLaserAnalyzer.event, edm::service::SimpleMemoryCheck::SignificantEvent.event, EcalLaserAnalyzer.event, EcalLaserAnalyzer2.event, FastTimerService::Timing.event, FastTimerService::TimingPerProcess.event, FastTimerService::SummaryPlots.event, FastTimerService::SummaryPlotsPerProcess.event, FastTimerService::SummaryProfiles.event, FastTimerService::SummaryProfilesPerProcess.event, JsonOutputProducer::JsonEvent.event, SummaryOutputProducer::Event.event, generateEDF.LumiInfo.events, Forest.events, HitEff.events, Node.events, ME::Header.events, looper.Looper.events, MatrixUtil.InputInfo.events, CosmicRateAnalyzer.events, options.HLTProcessOptions.events, JsonOutputProducer::JsonConfiguration.events, event.Event.setup, and looper.Looper.setup.

Referenced by ConfigBuilder.ConfigBuilder.addExtraStream(), ConfigBuilder.ConfigBuilder.completeInputCommand(), ConfigBuilder.ConfigBuilder.doNotInlineEventContent(), ConfigBuilder.ConfigBuilder.PrintAllModules.leave(), ConfigBuilder.ConfigBuilder.prepare_HLT(), ConfigBuilder.ConfigBuilder.prepare_LHE(), ConfigBuilder.ConfigBuilder.prepare_PATFILTER(), ConfigBuilder.ConfigBuilder.prepare_VALIDATION(), ConfigBuilder.ConfigBuilder.renameHLTprocessInSequence(), ConfigBuilder.ConfigBuilder.renameInputTagsInSequence(), and ConfigBuilder.ConfigBuilder.scheduleSequence().

222  def process(self, iEv ):
223  """Run event processing for all analyzers in the sequence.
224 
225  This function is called by self.loop,
226  but can also be called directly from
227  the python interpreter, to jump to a given event.
228  """
229  self.event = Event(iEv, self.events[iEv], self.setup)
230  self.iEvent = iEv
231  for i,analyzer in enumerate(self.analyzers):
232  if not analyzer.beginLoopCalled:
233  analyzer.beginLoop(self.setup)
234  start = timeit.default_timer()
235  if self.memReportFirstEvent >=0 and iEv >= self.memReportFirstEvent:
236  memNow=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
237  if memNow > self.memLast :
238  print "Mem Jump detected before analyzer %s at event %s. RSS(before,after,difference) %s %s %s "%( analyzer.name, iEv, self.memLast, memNow, memNow-self.memLast)
239  self.memLast=memNow
240  ret = analyzer.process( self.event )
241  if self.memReportFirstEvent >=0 and iEv >= self.memReportFirstEvent:
242  memNow=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
243  if memNow > self.memLast :
244  print "Mem Jump detected in analyzer %s at event %s. RSS(before,after,difference) %s %s %s "%( analyzer.name, iEv, self.memLast, memNow, memNow-self.memLast)
245  self.memLast=memNow
246  if self.timeReport:
247  self.timeReport[i]['events'] += 1
248  if self.timeReport[i]['events'] > 0:
249  self.timeReport[i]['time'] += timeit.default_timer() - start
250  if ret == False:
251  return (False, analyzer.name)
252  if iEv<self.nPrint:
253  self.logger.info( self.event.__str__() )
254  return (True, analyzer.name)
memReportFirstEvent
Definition: looper.py:90
def process
Definition: looper.py:221
def looper.Looper.write (   self)
Writes all analyzers.

See Analyzer.Write for more information.

Definition at line 255 of file looper.py.

References looper.Looper.analyzers, Config.Process.analyzers, event.Event.setup, and looper.Looper.setup.

256  def write(self):
257  """Writes all analyzers.
258 
259  See Analyzer.Write for more information.
260  """
261  for analyzer in self.analyzers:
262  analyzer.write(self.setup)
263  self.setup.close()
264 

Member Data Documentation

looper.Looper.analyzers

Definition at line 85 of file looper.py.

Referenced by looper.Looper.loop(), and looper.Looper.write().

looper.Looper.cfg_comp

Definition at line 83 of file looper.py.

Referenced by looper.Looper._build(), core.AutoFillTreeProducer.AutoFillTreeProducer.fillCoreVariables(), looper.Looper.loop(), and JetAnalyzer.JetAnalyzer.process().

looper.Looper.classes

Definition at line 84 of file looper.py.

looper.Looper.config

Definition at line 73 of file looper.py.

looper.Looper.event

Definition at line 117 of file looper.py.

Referenced by Types.EventID.cppID(), and looper.Looper.process().

looper.Looper.events

Definition at line 102 of file looper.py.

Referenced by eventsfwlite.Events.__getattr__(), eventsfwlite.Events.__init__(), looper.Looper.loop(), and looper.Looper.process().

looper.Looper.firstEvent

Definition at line 87 of file looper.py.

Referenced by looper.Looper.loop().

looper.Looper.iEvent

Definition at line 229 of file looper.py.

looper.Looper.logger

Definition at line 76 of file looper.py.

looper.Looper.memLast

Definition at line 91 of file looper.py.

looper.Looper.memReportFirstEvent

Definition at line 90 of file looper.py.

looper.Looper.name

Definition at line 74 of file looper.py.

Referenced by dirstructure.Directory.__create_pie_image(), dqm_interfaces.DirID.__eq__(), dirstructure.Directory.__get_full_path(), dirstructure.Comparison.__get_img_name(), dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), cuy.divideElement.__init__(), cuy.plotElement.__init__(), cuy.additionElement.__init__(), cuy.superimposeElement.__init__(), cuy.graphElement.__init__(), dirstructure.Comparison.__make_image(), dirstructure.Directory.__repr__(), dqm_interfaces.DirID.__repr__(), dirstructure.Comparison.__repr__(), config.CFG.__str__(), counter.Counter.__str__(), average.Average.__str__(), dirstructure.Directory.calcStats(), validation.Sample.digest(), python.rootplot.utilities.Hist.divide(), python.rootplot.utilities.Hist.divide_wilson(), utils.StatisticalTest.get_status(), production_tasks.Task.getname(), dataset.CMSDataset.getPrimaryDatasetEntries(), dataset.PrivateDataset.getPrimaryDatasetEntries(), VIDSelectorBase.VIDSelectorBase.initialize(), dirstructure.Directory.print_report(), dataset.BaseDataset.printInfo(), dataset.Dataset.printInfo(), production_tasks.MonitorJobs.run(), python.rootplot.utilities.Hist.TGraph(), python.rootplot.utilities.Hist.TH1F(), Vispa.Views.PropertyView.Property.valueChanged(), counter.Counter.write(), and average.Average.write().

looper.Looper.nEvents

Definition at line 86 of file looper.py.

Referenced by looper.Looper.loop().

looper.Looper.nPrint

Definition at line 88 of file looper.py.

looper.Looper.outDir

Definition at line 75 of file looper.py.

Referenced by looper.Looper._build().

looper.Looper.setup

Definition at line 125 of file looper.py.

Referenced by looper.Looper.loop(), looper.Looper.process(), and looper.Looper.write().

looper.Looper.start_time

Definition at line 179 of file looper.py.

Referenced by progressbar.ProgressBar.__next__(), and progressbar.ProgressBar.update().

looper.Looper.start_time_event

Definition at line 180 of file looper.py.

looper.Looper.timeReport

Definition at line 89 of file looper.py.