CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
cmsPerfSuiteHarvest Namespace Reference

Functions

def _eventContent_DEBUG
 
def assign_event_content_for_product
 
def exportSequences
 
def exportTimeSizeJob
 
def get_modules_sequences_relationships
 
def get_params
 
def process_timesize_dir
 
def searchTimeSizeFiles
 
def usage
 

Variables

 _TEST_RUN = False
 
dictionary candles = {}
 
tuple eventContentRules = parseEventContent.getTxtEventContentRules()
 
 EventContents_OK = False
 
string file_name = "%s___%s___%s___%s___%s___%s___%s.xml"
 
tuple now = datetime.datetime.now()
 
tuple p = parserPerfsuiteMetadata(os.getcwd())
 
dictionary pileups = {}
 
 release = None
 
tuple run_info = p.parseAll()
 
 Sequences_OK = False
 
dictionary steps = {}
 
tuple test_timing_report_log = re.compile("TimingReport.log$", re.IGNORECASE)
 
tuple xmldoc = minidom.Document()
 

Function Documentation

def cmsPerfSuiteHarvest._eventContent_DEBUG (   edm_report)
private

Definition at line 77 of file cmsPerfSuiteHarvest.py.

References python.multivaluedict.append(), join(), and parseEventContent.List_ECs_forProduct().

Referenced by process_timesize_dir().

77 
78 def _eventContent_DEBUG(edm_report):
79  # for testing / information
80  EC_count = {}
81  if not _TEST_RUN:
82  # count the products in event-content's
83  for prod in edm_report:
85  for ec in ecs:
86  if not EC_count.has_key(ec):
87  EC_count[ec] = []
88  EC_count[ec].append(prod)
89  #print out the statistics
90  for (ec, prods) in EC_count.items():
91  print "==== %s EVENT CONTENT: have %d items, the listing is: ===" % (ec, len(prods))
92  # list of products
93  print "\n *".join(["%(cpp_type)s_%(module_name)s_%(module_label)s" % prod for prod in prods])
94 
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def cmsPerfSuiteHarvest.assign_event_content_for_product (   product)
returns modified product by adding the event content relationship 

Definition at line 95 of file cmsPerfSuiteHarvest.py.

References join(), and parseEventContent.List_ECs_forProduct().

95 
97  """ returns modified product by adding the event content relationship """
98 
99  if not _TEST_RUN:
100  product["event_content"] = ",".join(parseEventContent.List_ECs_forProduct(product))
101  return product
102 
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def cmsPerfSuiteHarvest.exportSequences ( )
Exports the sequences to XML Doc 

Definition at line 235 of file cmsPerfSuiteHarvest.py.

References get_modules_sequences_relationships(), get_params(), and cmssw_exportdb_xml.xml_export_Sequences().

236 def exportSequences():
237  """ Exports the sequences to XML Doc """
238  try:
239  env_cmssw_version = os.environ["CMSSW_VERSION"]
240  except KeyError:
241  print "<<<<< ====== Error: cannot get CMSSW version [just integrity check for sequences]. \
242  Is the CMSSW environment initialized? (use cmsenv) ==== >>>>"
243  env_cmssw_version = None
244 
245  print " ==== exporting the sequences. loading files for currently loaded CMSSW version: %s, while the CMSSW we are currently harversting is %s ===" %(env_cmssw_version, release)
246  xml_export_Sequences(xml_doc = xmldoc, sequences = get_modules_sequences_relationships(), release=release)
247 
248 
def cmsPerfSuiteHarvest.exportTimeSizeJob (   path,
  timeSizeReport,
  runinfo 
)

Definition at line 108 of file cmsPerfSuiteHarvest.py.

References python.multivaluedict.append(), and python.rootplot.root2matplotlib.replace().

Referenced by process_timesize_dir().

109 def exportTimeSizeJob(path, timeSizeReport, runinfo):
110  candleLong = os.path.split(path)[1].replace("_TimeSize", "").replace("_PU", "")
111  jobID = timeSizeReport["jobID"]
112  print candleLong
113 
114  #search for a run Test to which could belong our JOB
115  found = False
116  if runinfo['TestResults'].has_key('TimeSize'):
117  for result in runinfo['TestResults']['TimeSize']:
118  #print result
119  """ If this is the testResult which fits TimeSize job """
120  #TODO: we do not check teh step when assigning because of the different names, check if this is really OK. make a decission which step name to use later, long or short one
121  #and jobID["step"] in result['steps'].split(parserPerfsuiteMetadata._LINE_SEPARATOR)
122  if result['candle'] == candleLong and jobID["pileup_type"] == result['pileup_type'] and jobID["conditions"] == result['conditions'] and jobID["event_content"] == result['event_content']:
123  #print result
124  if not result.has_key("jobs"):
125  result['jobs'] = []
126  result['jobs'].append(timeSizeReport)
127  found = True
128  break
129 
130  if not found:
131  print "============ (almost) ERROR: NOT FOUND THE ENTRY in cmsPerfSuite.log, exporting as separate entry ======== "
132  print "JOB ID: %s " % str(jobID)
133  print " ====================== "
134  runinfo['unrecognized_jobs'].append(timeSizeReport)
135  #export_xml(xml_doc = xmldoc, **timeSizeReport)
136 
def cmsPerfSuiteHarvest.get_modules_sequences_relationships ( )

Definition at line 103 of file cmsPerfSuiteHarvest.py.

References ModuleToSequenceAssign.assignModulesToSeqs(), and join().

Referenced by exportSequences().

105  (sequenceWithModules, sequenceWithModulesString) =ModuleToSequenceAssign.assignModulesToSeqs()
106  return [{"name": seq, "modules": ",".join(modules)} for (seq, modules) in sequenceWithModulesString.items()]
107 
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def cmsPerfSuiteHarvest.get_params (   argv)
Returns the version of CMSSW to be used which it is taken from:
* command line parameter or 
* environment variable 
in case of error returns None

And also the directory to put the xml files to: if none --> returns ""
try to get the version for command line argument 

Definition at line 40 of file cmsPerfSuiteHarvest.py.

Referenced by exportSequences().

40 
41 def get_params(argv):
42  """
43  Returns the version of CMSSW to be used which it is taken from:
44  * command line parameter or
45  * environment variable
46  in case of error returns None
47 
48  And also the directory to put the xml files to: if none --> returns ""
49  """
50 
51  """ try to get the version for command line argument """
52  #print argv
53  #FIXME: this should be rewritten using getopt properly
54  version = None
55  #xml_dir = "cmsperfvm:/data/projects/conf/PerfSuiteDB/xml_dropbox" #Set this as default (assume change in write_xml to write to remote machines)
56  #NB write_xml is in Validation/Performance/python/cmssw_exportdb_xml.py
57  #Setting the default to write to a local directory:
58  xml_dir="PerfSuiteDBData"
59  try:
60  opts, args = getopt.getopt(argv[1:], "v:", ["version=", "outdir="])
61  except getopt.GetoptError, e:
62  print e
63  for opt, arg in opts:
64  if opt in ("-v", "--version"):
65  version = arg
66  if opt == "--outdir":
67  xml_dir = arg
68 
69  """ if not get it from environment string """
70  if not version:
71  try:
72  version = os.environ["CMSSW_VERSION"]
73  except KeyError:
74  pass
75 
76  return (version, xml_dir)
def cmsPerfSuiteHarvest.process_timesize_dir (   path,
  runinfo 
)

Definition at line 137 of file cmsPerfSuiteHarvest.py.

References _eventContent_DEBUG(), python.multivaluedict.dict, edm::hlt.Exception, exportTimeSizeJob(), parserEdmSize.getEdmReport(), FileNamesHelper.getJobID_fromTimeReportLogName(), FileNamesHelper.getRootFileSize(), parserTimingReport.loadTimeLog(), python.multivaluedict.map(), parserTimingReport.processModuleTimeLogData(), FileNamesHelper.read_ConfigurationFromSimulationCandles(), and FileNamesHelper.read_SimulationCandles().

Referenced by searchTimeSizeFiles().

138 def process_timesize_dir(path, runinfo):
139  global release,event_content,conditions
140  """ if the release is not provided explicitly we take it from the Simulation candles file """
141  if (not release):
142  release_fromlogfile = read_SimulationCandles(path)
143  release = release_fromlogfile
144  print "release from simulation candles: %s" % release
145 
146  if (not release):
147  # TODO: raise exception!
148  raise Exception("the release was not found!")
149 
150 
151  """ process the TimingReport log files """
152 
153  # get the file list
154  files = os.listdir(path)
155  timing_report_files = [os.path.join(path, f) for f in files
156  if test_timing_report_log.search(f)
157  and os.path.isfile(os.path.join(path, f)) ]
158 
159  # print timing_report_files
160  for timelog_f in timing_report_files:
161  print "\nProcessing file: %s" % timelog_f
162  print "------- "
163 
164  jobID = getJobID_fromTimeReportLogName(os.path.join(path, timelog_f))
165  print "jobID: %s" % str(jobID)
166  (candle, step, pileup_type, conditions, event_content) = jobID
167  jobID = dict(zip(("candle", "step", "pileup_type", "conditions", "event_content"), jobID))
168  print "Dictionary based jobID %s: " % str(jobID)
169 
170  #if any of jobID fields except (isPILEUP) is empty we discard the job as all those are the jobID keys and we must have them
171  discard = len([key for key, value in jobID.items() if key != "pileup_type" and not value])
172  if discard:
173  print " ====================== The job HAS BEEN DISCARDED =============== "
174  print " NOT ALL DATA WAS AVAILABLE "
175  print " JOB ID = %s " % str(jobID)
176  print " ======================= end ===================================== "
177  continue
178 
179  num_events = read_ConfigurationFromSimulationCandles(path = path, step = step, is_pileup = pileup_type)["num_events"]
180  # TODO: automaticaly detect type of report file!!!
181  (mod_timelog, evt_timelog, rss_data, vsize_data) =loadTimeLog(timelog_f)
182 
183  mod_timelog= processModuleTimeLogData(mod_timelog, groupBy = "module_name")
184  print "Number of modules grouped by (module_label+module_name): %s" % len(mod_timelog)
185 
186  # add to the list to generate the readable filename :)
187  steps[step] = 1
188  candles[candle] = 1
189  if pileup_type=="":
190  pileups["NoPileUp"]=1
191  else:
192  pileups[pileup_type] = 1
193 
194  # root file size (number)
195  root_file_size = getRootFileSize(path = path, candle = candle, step = step)
196 
197  #EdmSize
198  edm_report = parserEdmSize.getEdmReport(path = path, candle = candle, step = step)
199  if edm_report != False:
200  try:
201  # add event content data
202  edm_report = map(assign_event_content_for_product, edm_report)
203  # for testing / imformation
204  _eventContent_DEBUG(edm_report)
205  except Exception, e:
206  print e
207 
208 
209  timeSizeReport = {
210  "jobID":jobID,
211  "release": release,
212  "timelog_result": (mod_timelog, evt_timelog, rss_data, vsize_data),
213  "metadata": {"root_file_size": root_file_size, "num_events": num_events},
214  "edmSize_result": edm_report
215  }
216 
217  # export to xml: actualy exporting gets suspended and put into runinfo
218  exportTimeSizeJob(path, timeSizeReport, runinfo)
219 
#TimeSize
def getJobID_fromTimeReportLogName
def processModuleTimeLogData
mod_data[&quot;stats&quot;] =calc_MinMaxAvgRMS(f_time = lambda x: x[&quot;time&quot;], f_evt_num = lambda x: x[&quot;event_num...
def read_ConfigurationFromSimulationCandles
def cmsPerfSuiteHarvest.searchTimeSizeFiles (   runinfo)
so far we will use the current dir to search in 

Definition at line 220 of file cmsPerfSuiteHarvest.py.

References process_timesize_dir().

221 def searchTimeSizeFiles(runinfo):
222  """ so far we will use the current dir to search in """
223  path = os.getcwd()
224  #print path
225  print 'full path =', os.path.abspath(path)
226 
227  files = os.listdir(path)
228 
229  test_timeSizeDirs = re.compile("_TimeSize$", re.IGNORECASE)
230  timesize_dirs = [os.path.join(path, f) for f in files if test_timeSizeDirs.search(f) and os.path.isdir(os.path.join(path, f))]
231 
232  for timesize_dir in timesize_dirs:
233  # print timesize_dir
234  process_timesize_dir(timesize_dir, runinfo)
def cmsPerfSuiteHarvest.usage (   argv)

Definition at line 26 of file cmsPerfSuiteHarvest.py.

26 
27 def usage(argv):
28  script = argv[0]
29  return """
30  Usage: %(script)s [-v cmssw_version] [--version=cmssw_version]
31 
32  if the cmssw version is in the system's environment (after running cmsenv):
33  $ %(script)s
34 
35  otherwise one must specify the cmssw version:
36  $ %(script)s --version=CMSSW_3_2_0
37  $ %(script)s -v CMSSW_3_2_0
38 
39  """ % locals()

Variable Documentation

cmsPerfSuiteHarvest._TEST_RUN = False

Definition at line 14 of file cmsPerfSuiteHarvest.py.

dictionary cmsPerfSuiteHarvest.candles = {}

Definition at line 23 of file cmsPerfSuiteHarvest.py.

tuple cmsPerfSuiteHarvest.eventContentRules = parseEventContent.getTxtEventContentRules()

Definition at line 298 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.EventContents_OK = False

Definition at line 270 of file cmsPerfSuiteHarvest.py.

string cmsPerfSuiteHarvest.file_name = "%s___%s___%s___%s___%s___%s___%s.xml"

Definition at line 308 of file cmsPerfSuiteHarvest.py.

Referenced by HcalLutManager.create_lut_loader(), SiPixelHistoPlotter.createStaticPlot(), SiStripHistoPlotter.createStaticPlot(), DTTPGLutFile.open(), L1TriggerLutFile.open(), TEcnaRead.ReadAveragedHighFrequencyNoise(), TEcnaRead.ReadAveragedLowFrequencyNoise(), TEcnaRead.ReadAveragedMeanOfCorrelationsBetweenSamples(), TEcnaRead.ReadAveragedPedestals(), TEcnaRead.ReadAveragedSigmaOfCorrelationsBetweenSamples(), TEcnaRead.ReadAveragedTotalNoise(), TEcnaRead.ReadCorrelationsBetweenSamples(), TEcnaRead.ReadCovariancesBetweenSamples(), TEcnaRead.ReadHighFrequencyCorrelationsBetweenChannels(), TEcnaRead.ReadHighFrequencyCovariancesBetweenChannels(), TEcnaRead.ReadHighFrequencyMeanCorrelationsBetweenStins(), TEcnaRead.ReadHighFrequencyNoise(), TEcnaRead.ReadLowFrequencyCorrelationsBetweenChannels(), TEcnaRead.ReadLowFrequencyCovariancesBetweenChannels(), TEcnaRead.ReadLowFrequencyMeanCorrelationsBetweenStins(), TEcnaRead.ReadLowFrequencyNoise(), TEcnaRead.ReadMeanOfCorrelationsBetweenSamples(), TEcnaRead.ReadNumberOfEventsForSamples(), TEcnaRead.ReadPedestals(), TEcnaRead.ReadRelevantCorrelationsBetweenSamples(), TEcnaRead.ReadRootFileHeader(), TEcnaRead.ReadSampleMeans(), TEcnaRead.ReadSampleSigmas(), TEcnaRead.ReadSampleValues(), TEcnaRead.ReadSampleValuesSameFile(), TEcnaRead.ReadSigmaOfCorrelationsBetweenSamples(), TEcnaRead.ReadStinNumbers(), TEcnaRead.ReadTotalNoise(), and TEcnaRun.WriteRootFile().

tuple cmsPerfSuiteHarvest.now = datetime.datetime.now()

Definition at line 305 of file cmsPerfSuiteHarvest.py.

Referenced by edm::CountAndLimit.add(), evf::EvFRecordInserter.analyze(), HcalLuttoDB.beginJob(), evf::FUResourceTable.buildResource(), stor::MonitorCollection.calculateStatistics(), smproxy::StatisticsReporter.calculateStatistics(), stor::StatisticsReporter.calculateStatistics(), condbon.cdbon_write(), stor::DiskWriter.checkForFileTimeOuts(), smproxy::DataManager.checkForStaleConsumers(), stor::EventDistributor.checkForStaleConsumers(), evf::iDie.detailsTable(), DTHVStatusHandler.dumpSnapshot(), Fit.getFitQuality(), DTHVStatusHandler.getNewObjects(), stor::QueueCollection< T >.getQueue(), HtrXmlPatternWriter.HtrXmlPatternWriter(), cond::Logger.logFailedOperationNow(), DQMNet.logme(), cond::Logger.logOperationNow(), Fit.multiplyMatrices(), operator<<(), evf::iDie.postEntry(), Fit.PropagateErrors(), evf::Vulture.prowling(), smproxy::StatisticsReporter.reset(), stor::StatisticsReporter.reset(), lumi::CMSRunSummaryDummy2DB.retrieveData(), DQMNet.run(), ALIUtils.set_time_now(), edm::TimeoutPoolOutputModule.shouldWeCloseFile(), evf::Vulture.startProwling(), evf::FUResourceBroker.stopping(), and stor::EventDistributor.tagCompleteEventForQueues().

tuple cmsPerfSuiteHarvest.p = parserPerfsuiteMetadata(os.getcwd())

Definition at line 264 of file cmsPerfSuiteHarvest.py.

dictionary cmsPerfSuiteHarvest.pileups = {}

Definition at line 24 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.release = None

Definition at line 21 of file cmsPerfSuiteHarvest.py.

Referenced by stor::XHTMLMaker.addNode(), stor::XHTMLMaker.addText(), ConfigurationDBHandler.cvt2String(), L1GtTriggerMenuXmlParser.findXMLChild(), XMLDOMBlock.getNewDocument(), L1GtTriggerMenuXmlParser.getXMLAttribute(), L1GtTriggerMenuXmlParser.getXMLTextValue(), MonitorXMLParser.handleElement(), cond::XMLAuthenticationService::XMLAuthenticationService.initialize(), L1GtTriggerMenuXmlParser.initXML(), MonitorXMLParser.load(), MonitorXMLParser.MonitorXMLParser(), MuonAlignmentInputXML.newAlignableMuon(), stor::XHTMLMaker.out(), XMLDOMBlock.parse(), L1GtTriggerMenuXmlParser.parseConditions(), MuonAlignmentInputXML.parseDouble(), SummaryPlotXmlParser.parseXML(), cond::XMLAuthenticationService::XMLAuthenticationService.processFile(), CalibrationXML.readAttribute(), XMLProcessor.serializeDOM(), stor::XHTMLMaker.start(), cscdqm::XMLFileErrorHandler.warning(), L1GtTriggerMenuXmlParser.workXML(), CalibrationXML.writeAttribute(), stor::XHTMLMaker.XHTMLMaker(), xMLCh2String(), XMLDOMBlock.XMLDOMBlock(), ConfigurationDBHandler.~ConfigurationDBHandler(), LutXml.~LutXml(), MonitorXMLParser.~MonitorXMLParser(), MuonAlignmentInputXML.~MuonAlignmentInputXML(), edm::SingleConsumerQ::OperateBuffer< T >.~OperateBuffer(), edm::EventBuffer::OperateBuffer< T >.~OperateBuffer(), StrX.~StrX(), TagNames.~TagNames(), cscdqm::XercesStringTranscoder.~XercesStringTranscoder(), XMLSimpleStr.~XMLSimpleStr(), lhef::XMLSimpleStr.~XMLSimpleStr(), XMLUniStr.~XMLUniStr(), lhef::XMLUniStr.~XMLUniStr(), and XStr.~XStr().

tuple cmsPerfSuiteHarvest.run_info = p.parseAll()

Definition at line 265 of file cmsPerfSuiteHarvest.py.

cmsPerfSuiteHarvest.Sequences_OK = False

Definition at line 269 of file cmsPerfSuiteHarvest.py.

dictionary cmsPerfSuiteHarvest.steps = {}

Definition at line 22 of file cmsPerfSuiteHarvest.py.

Referenced by TrackingMaterialPlotter.fill_gradient(), XHistogram.splitSegment(), CosmicMuonUtilities.stepPropagate(), and CSCGasCollisions.writeSummary().

tuple cmsPerfSuiteHarvest.test_timing_report_log = re.compile("TimingReport.log$", re.IGNORECASE)

Definition at line 17 of file cmsPerfSuiteHarvest.py.

tuple cmsPerfSuiteHarvest.xmldoc = minidom.Document()

Definition at line 20 of file cmsPerfSuiteHarvest.py.