test
CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions
cmssw_exportdb_xml Namespace Reference

Functions

def _getXMLNode
 
def createNode
 
def export_xml
 
def export_xml_ig
 
def export_xml_memcheck
 
def exportECRules
 
def exportRunInfo
 
def initXML
 
def write_xml
 
def xml_delete_children
 
def xml_export_EdmRecord
 
def xml_export_EventRssRecord
 
def xml_export_EventTimeRecord
 
def xml_export_EventVsizeRecord
 
def xml_export_IgSummary
 
def xml_export_Memcheck
 
def xml_export_ModuleTimeRecord
 
def xml_export_SequenceRecord
 
def xml_export_Sequences
 
def xml_init_Sequences
 

Function Documentation

def cmssw_exportdb_xml._getXMLNode (   xml_doc)
private
opens existing or creates a new XML object 

Definition at line 104 of file cmssw_exportdb_xml.py.

References createNode().

Referenced by export_xml(), export_xml_ig(), export_xml_memcheck(), exportECRules(), and exportRunInfo().

105 def _getXMLNode(xml_doc):
106  """ opens existing or creates a new XML object """
107  try:
108  node_xml = xml_doc.getElementsByTagName("xml")[0]
109  except IndexError:
110  #doc = minidom.Document()
111  node_xml = createNode(xml_doc, "xml")
112  return node_xml
113 
114 
115 
def cmssw_exportdb_xml.createNode (   xml_doc,
  node_name,
  values = {},
  parent = None 
)

Definition at line 4 of file cmssw_exportdb_xml.py.

Referenced by _getXMLNode(), export_xml(), export_xml_ig(), export_xml_memcheck(), exportECRules(), exportRunInfo(), initXML(), xml_export_EdmRecord(), xml_export_EventRssRecord(), xml_export_EventTimeRecord(), xml_export_EventVsizeRecord(), xml_export_IgSummary(), xml_export_Memcheck(), xml_export_ModuleTimeRecord(), xml_export_SequenceRecord(), and xml_init_Sequences().

4 
5 def createNode(xml_doc, node_name, values = {}, parent = None):
6  if (parent == None):
7  parent = xml_doc
8  #create node
9  node = xml_doc.createElement(node_name)
10  # assign the values
11  for (key, value) in values.items():
12  node.setAttribute(key, str(value))
13  parent.appendChild(node)
14 
15  return node
16 
def cmssw_exportdb_xml.export_xml (   release,
  jobID,
  timelog_result,
  xml_doc,
  metadata = None,
  edmSize_result = None,
  parentNode = None 
)
jobID is a dictionary now ! 

Definition at line 125 of file cmssw_exportdb_xml.py.

References _getXMLNode(), createNode(), xml_export_EdmRecord(), xml_export_EventRssRecord(), xml_export_EventTimeRecord(), xml_export_EventVsizeRecord(), and xml_export_ModuleTimeRecord().

Referenced by exportRunInfo(), parserTimingReport.manual_run(), and parserTimingReport.perf_profile().

126 def export_xml(release, jobID, timelog_result, xml_doc, metadata = None, edmSize_result =None, parentNode = None):
127  """ jobID is a dictionary now ! """
128 
129  if not parentNode:
130  #get the root XML node
131  parentNode = _getXMLNode(xml_doc)
132 
133  #create jobStats node
134  values=jobID
135  values.update({"release": release})
136  if (metadata):
137  values.update(metadata)
138 
139  # we create a new XML element having all the statistics data for our candle, step, release
140  jobStatsNode = createNode(node_name= "jobStats", xml_doc=xml_doc, parent=parentNode,
141  values=values)
142 
143  #TODO: load current module data or delete
144 
145  (mod_timelog_result, evt_timelog_result, rss_result, vsize_result) = timelog_result
146 
147  # modules data
148  for (mod_name, mod_time_result) in mod_timelog_result.items():
149  xml_export_ModuleTimeRecord(mod_time_result, jobStatsNode, xml_doc)
150 
151  """ #TODO: actualy we're not supposed to have any of the old statistics
152  # clean it up if there are old stats of EventTime,RSS,VSIZE
153  xml_delete_children(jobStatsNode, "EventTime")
154  xml_delete_children(jobStatsNode, "EventRSS")
155  xml_delete_children(jobStatsNode, "EventVSIZE")
156  xml_delete_children(jobStatsNode, "EdmSize") """
157 
158  #events data - so far only total time per event
159  if (evt_timelog_result):
160  for evt_time_item in evt_timelog_result:
161  xml_export_EventTimeRecord(evt_time_item, jobStatsNode, xml_doc)
162 
163  # rss
164  if (rss_result):
165  for evt_time_item in rss_result:
166  xml_export_EventRssRecord(evt_time_item, jobStatsNode, xml_doc)
167 
168  # vsize
169  if (vsize_result):
170  for evt_time_item in vsize_result:
171  xml_export_EventVsizeRecord(evt_time_item, jobStatsNode, xml_doc)
172  # edmSize
173  if (edmSize_result):
174  for edmItem in edmSize_result:
175  xml_export_EdmRecord(edmItem, jobStatsNode, xml_doc)
def cmssw_exportdb_xml.export_xml_ig (   release,
  jobID,
  igprof_result,
  xml_doc,
  metadata = None,
  parentNode = None 
)
jobID is a dictionary now ! 

Definition at line 176 of file cmssw_exportdb_xml.py.

References _getXMLNode(), createNode(), and xml_export_IgSummary().

Referenced by exportRunInfo().

177 def export_xml_ig(release, jobID, igprof_result, xml_doc, metadata = None, parentNode = None):
178  """ jobID is a dictionary now ! """
179 
180  if not parentNode:
181  #get the root XML node
182  parentNode = _getXMLNode(xml_doc)
183 
184  #create jobStats node
185  values=jobID
186  values.update({"release": release})
187  if (metadata):
188  values.update(metadata)
189 
190  # we create a new XML element having all the statistics data for our candle, step, release
191  jobStatsNode = createNode(node_name= "jobStats", xml_doc=xml_doc, parent=parentNode,
192  values=values)
193 
194  for igsumm in igprof_result:
195  xml_export_IgSummary(igsumm, jobStatsNode, xml_doc)
def cmssw_exportdb_xml.export_xml_memcheck (   release,
  jobID,
  memcheck_errors,
  xml_doc,
  metadata = None,
  parentNode = None 
)

Definition at line 196 of file cmssw_exportdb_xml.py.

References _getXMLNode(), createNode(), and xml_export_Memcheck().

Referenced by exportRunInfo().

197 def export_xml_memcheck(release, jobID, memcheck_errors, xml_doc, metadata = None, parentNode = None):
198  if not parentNode:
199  parentNode = _getXMLNode(xml_doc)
200 
201  values=jobID
202  values.update({"release": release})
203  if (metadata):
204  values.update(metadata)
205 
206  # we create a new XML element having all the statistics data for our candle, step, release
207  jobStatsNode = createNode(node_name= "jobStats", xml_doc=xml_doc, parent=parentNode,
208  values=values)
209 
210  xml_export_Memcheck(memcheck_errors, jobStatsNode, xml_doc)
def cmssw_exportdb_xml.exportECRules (   xml_doc,
  rules 
)

Definition at line 282 of file cmssw_exportdb_xml.py.

References _getXMLNode(), and createNode().

283 def exportECRules(xml_doc, rules):
284  node_xml = _getXMLNode(xml_doc)
285  runInfoNode = createNode(node_name= "EventContentRules", xml_doc=xml_doc, parent=node_xml,
286  values=rules)
def cmssw_exportdb_xml.exportRunInfo (   xml_doc,
  run_info,
  release = None,
  print_out = False 
)

Definition at line 211 of file cmssw_exportdb_xml.py.

References _getXMLNode(), createNode(), export_xml(), export_xml_ig(), export_xml_memcheck(), and mps_monitormerge.items.

212 def exportRunInfo(xml_doc, run_info, release = None, print_out = False):
213  node_xml = _getXMLNode(xml_doc)
214  #get the simple string values
215  str_values = run_info["General"]
216 
217  #if we have the forced release name (e.g. icludes some special tags for testing but not official release)
218  # so TODO: probably the test_release_based_on string would be the same so we still save the (original) test release based string
219 
220  if release:
221  str_values["release"] = release
222  else:
223  str_values["release"] = str_values["test_release_based_on"]
224 
225  runInfoNode = createNode(node_name= "RunInfo", xml_doc=xml_doc, parent=node_xml,
226  values=str_values)
227  #create nodes for TestResults:
228  for (testName, result) in run_info["TestResults"].items():
229  #either we have one node or multiple ones (if list)
230  if type(result) == list:
231  for result_item in result:
232  result_item.update({"testname": testName})
233 
234  #We have JOBS so FAR only for TimeSize which we represent as a list
235  jobs = []
236  #we don't want jobs to be dumped as string
237  if "jobs" in result_item:
238  jobs = result_item["jobs"]
239  del result_item["jobs"]
240 
241  testNode = createNode(node_name="testResult", xml_doc=xml_doc, parent=runInfoNode, values=result_item)
242 
243 
244  for job in jobs:
245  #print job
246  if testName == "TimeSize":
247  export_xml(xml_doc = xml_doc, parentNode = testNode, **job)
248  elif testName == "IgProf_Mem":
249  export_xml_ig(xml_doc = xml_doc, parentNode = testNode, **job)
250  elif testName == "IgProf_Perf":
251  export_xml_ig(xml_doc = xml_doc, parentNode = testNode, **job)
252  elif testName == "Memcheck":
253  export_xml_memcheck(xml_doc = xml_doc, parentNode = testNode, **job)
254  else:
255  result.update({"testname": testName})
256  createNode(node_name="testResult", xml_doc=xml_doc, parent=runInfoNode, values=result)
257 
258  #DO we have some unrecognized JOBS?
259  if len(run_info['unrecognized_jobs']):
260  unrecognizedJobsNode = createNode(node_name="Unrecognized_JOBS", xml_doc=xml_doc, parent=runInfoNode, values={})
261 
262  for job in run_info['unrecognized_jobs']:
263  #print job
264  testName = job["metadata"]["testname"]
265  if testName == "TimeSize":
266  export_xml(xml_doc = xml_doc, parentNode = unrecognizedJobsNode, **job)
267  elif testName == "IgProf_Mem":
268  export_xml_ig(xml_doc = xml_doc, parentNode = unrecognizedJobsNode, **job)
269  elif testName == "IgProf_Perf":
270  export_xml_ig(xml_doc = xml_doc, parentNode = unrecognizedJobsNode, **job)
271  elif testName == "Memcheck":
272  export_xml_memcheck(xml_doc = xml_doc, parentNode = unrecognizedJobsNode, **job)
273 
274 
275  #cmsSciMark
276  cmsSciMarkNode = createNode(node_name="cmsSciMarks", xml_doc=xml_doc, parent=runInfoNode, values = {})
277  for csiMark in run_info["cmsSciMark"]:
278  #print csiMark
279  createNode(node_name="cmsSciMark", xml_doc=xml_doc, parent=cmsSciMarkNode, values=csiMark)
280  if print_out:
281  print xml_doc.toprettyxml(indent="\t")
def cmssw_exportdb_xml.initXML (   xmldoc)
opens existing or creates a new XML file 
    
  ---- one of the erliest functions - quite nasty looking :)

Definition at line 17 of file cmssw_exportdb_xml.py.

References createNode().

17 
18 def initXML(xmldoc):
19  """ opens existing or creates a new XML file
20 
21  ---- one of the erliest functions - quite nasty looking :)"""
22 
23  try:
24  node_xml = xmldoc.getElementsByTagName("xml")[0]
25  except IndexError:
26  #doc = minidom.Document()
27  node_xml = createNode(xmldoc, "xml")
28 
29  return node_xml
def cmssw_exportdb_xml.write_xml (   xml_doc,
  remotedir,
  xmlFileName 
)

Definition at line 287 of file cmssw_exportdb_xml.py.

288 def write_xml(xml_doc, remotedir,xmlFileName):
289  xml = xml_doc.toprettyxml(indent=" ")
290 
291  # return xml as string (if requested)
292  if (xmlFileName == ""):
293  return xml
294  #Adding a modification to make sure the file is written in /tmp/$USER_perfsuite_xml dir (to allow any used to harvest any workdir without permission issues.
295  tmp_dir="/tmp/%s_perfsuite_xml"%os.getenv("USER")
296  if not os.path.exists(tmp_dir):
297  os.system("mkdir %s"%tmp_dir)
298  xmlFileName=os.path.join(tmp_dir,xmlFileName)
299  # or save that as file
300  out = open(xmlFileName, "w")
301  #print xml locally
302  out.write(xml)
303  out.close()
304  print "Now copying %s to stage directory %s..."%(xmlFileName,remotedir)
305 
306  #FIXME: Here we could decide to archive it on CASTOR on a dedicated directory
307  if ":" in remotedir: #CAVEAT: since we report the timestamp as part of the xml filename we need to be careful..
308  (host,dir)=remotedir.split(":")
309  #change to the directory to avoid tar replicating /tmp/$USER_perfsuite_xml/ directory structure remotely:
310  copy_cmd='cd %s; tar cf - %s|ssh %s "cd %s;tar xf -"'%(tmp_dir,os.path.basename(xmlFileName),host,dir)
311  else:
312  if not os.path.exists(remotedir):
313  os.system("mkdir %s"%remotedir)
314  #tarpipe_cmd='cd %s;tar cf - %s|(cd %s;tar xf -)'%(tmp_dir,os.path.basename(xmlFileName),remotedir)
315  copy_cmd='cp -pR %s/%s %s'%(tmp_dir,os.path.basename(xmlFileName),remotedir)
316  try:
317  print copy_cmd
318  os.system(copy_cmd)
319  print "Successfully copied XML report %s to stage directory %s"%(xmlFileName,remotedir)
320  except Exception as e :
321  print "Issues with copying XML report %s to stage directory %s!\n%s"%(xmlFileName,remotedir,str(e))
322  #os.system("cd -") #just in case...
323 
324 
def cmssw_exportdb_xml.xml_delete_children (   nodes,
  child_name 
)

Definition at line 30 of file cmssw_exportdb_xml.py.

Referenced by xml_export_Sequences().

30 
31 def xml_delete_children(nodes, child_name):
32  for x in nodes.getElementsByTagName(child_name):
33  nodes.removeChild(x)
34 
def cmssw_exportdb_xml.xml_export_EdmRecord (   data,
  curr_stat_node,
  xml_doc 
)

Definition at line 64 of file cmssw_exportdb_xml.py.

References createNode().

Referenced by export_xml().

64 
65 def xml_export_EdmRecord(data, curr_stat_node, xml_doc):
66  event_node =createNode(xml_doc, "EdmSize", values = data, parent = curr_stat_node)
def cmssw_exportdb_xml.xml_export_EventRssRecord (   evt_time_data,
  curr_stat_node,
  xml_doc 
)

Definition at line 54 of file cmssw_exportdb_xml.py.

References createNode().

Referenced by export_xml().

54 
55 def xml_export_EventRssRecord(evt_time_data, curr_stat_node, xml_doc):
56  #print "a"+str(evt_time_data)
57  (evt_num, evt_time) = evt_time_data
58  event_node =createNode(xml_doc, "EventRSS", values = {"evt_num": evt_num, "time": evt_time}, parent = curr_stat_node)
def cmssw_exportdb_xml.xml_export_EventTimeRecord (   evt_time_data,
  curr_stat_node,
  xml_doc 
)

Definition at line 48 of file cmssw_exportdb_xml.py.

References createNode().

Referenced by export_xml().

48 
49 def xml_export_EventTimeRecord(evt_time_data, curr_stat_node, xml_doc):
50  #print "a"+str(evt_time_data)
51  (evt_num, evt_time) = evt_time_data
52 
53  event_node =createNode(xml_doc, "EventTime", values = {"evt_num": evt_num, "time": evt_time}, parent = curr_stat_node)
def cmssw_exportdb_xml.xml_export_EventVsizeRecord (   evt_time_data,
  curr_stat_node,
  xml_doc 
)

Definition at line 59 of file cmssw_exportdb_xml.py.

References createNode().

Referenced by export_xml().

59 
60 def xml_export_EventVsizeRecord(evt_time_data, curr_stat_node, xml_doc):
61  #print "a"+str(evt_time_data)
62  (evt_num, evt_time) = evt_time_data
63  event_node =createNode(xml_doc, "EventVSIZE", values = {"evt_num": evt_num, "time": evt_time}, parent = curr_stat_node)
def cmssw_exportdb_xml.xml_export_IgSummary (   data,
  curr_stat_node,
  xml_doc 
)

Definition at line 67 of file cmssw_exportdb_xml.py.

References createNode().

Referenced by export_xml_ig().

67 
68 def xml_export_IgSummary(data, curr_stat_node, xml_doc):
69  event_node =createNode(xml_doc, "IgSummary", values = data, parent = curr_stat_node)
def cmssw_exportdb_xml.xml_export_Memcheck (   data,
  curr_stat_node,
  xml_doc 
)

Definition at line 70 of file cmssw_exportdb_xml.py.

References createNode().

Referenced by export_xml_memcheck().

70 
71 def xml_export_Memcheck(data, curr_stat_node, xml_doc):
72  event_node =createNode(xml_doc, "Memcheck", values = data, parent = curr_stat_node)
73 
def cmssw_exportdb_xml.xml_export_ModuleTimeRecord (   data_dict,
  curr_stat_node,
  xml_doc 
)

Definition at line 35 of file cmssw_exportdb_xml.py.

References createNode().

Referenced by export_xml().

35 
36 def xml_export_ModuleTimeRecord(data_dict, curr_stat_node, xml_doc):
37  #print "Dict: "+str(data_dict)
38  # create a module tag to be used to put the data into
39  module_node =createNode(xml_doc, "Module", values = data_dict, parent = curr_stat_node)
40 
41  # clean it up if there are old stats of ModuleTime, now there shouldn't be any anymore!
42  #xml_delete_children(module_node, "ModuleTime")
43 
44  # we create a new XML element having the statistics data
45  moduletime_stat =createNode(xml_doc, "ModuleTime", values = data_dict["stats"], parent = module_node)
46 
47  module_node.appendChild(moduletime_stat)
def cmssw_exportdb_xml.xml_export_SequenceRecord (   data,
  curr_seq_node,
  xml_doc 
)

Definition at line 74 of file cmssw_exportdb_xml.py.

References createNode().

Referenced by xml_export_Sequences().

74 
75 def xml_export_SequenceRecord(data, curr_seq_node, xml_doc):
76  seq_node =createNode(xml_doc, "Sequence", values = data, parent = curr_seq_node)
def cmssw_exportdb_xml.xml_export_Sequences (   xml_doc,
  sequences,
  release 
)

Definition at line 116 of file cmssw_exportdb_xml.py.

References xml_delete_children(), xml_export_SequenceRecord(), and xml_init_Sequences().

Referenced by cmsPerfSuiteHarvest.exportSequences().

117 def xml_export_Sequences(xml_doc, sequences, release):
118  currentNode = xml_init_Sequences(xml_doc, release)
119  xml_delete_children(currentNode, "Sequence")
120 
121  if (sequences):
122  for seq in sequences:
123  xml_export_SequenceRecord(data=seq, curr_seq_node=currentNode, xml_doc=xml_doc)
124 
def cmssw_exportdb_xml.xml_init_Sequences (   xml_doc,
  release 
)
opens existing or creates a new XML file 
    returns the (existing or created) unique element for statiscs

Definition at line 77 of file cmssw_exportdb_xml.py.

References createNode().

Referenced by xml_export_Sequences().

77 
78 def xml_init_Sequences(xml_doc, release):
79  #TODO: this was copied from init_XML - it should not be like that - refactor
80  """ opens existing or creates a new XML file
81  returns the (existing or created) unique element for statiscs"""
82  try:
83  node_xml = xml_doc.getElementsByTagName("xml")[0]
84  except IndexError:
85  #doc = minidom.Document()
86  node_xml = createNode(xml_doc, "xml")
87 
88  nodes =node_xml.getElementsByTagName("Sequences")
89 
90  currentNode = [node for node in nodes
91  if (release ==node.attributes["release"].value)]
92  #TODO: end of copied
93 
94  if (currentNode):
95  currentNode = currentNode[0]
96  #TODO: this part looks nasty
97  else:
98  values={"release": release}
99  # we create a new XML element having all the statistics data for our candle, step, release
100  currentNode = createNode(node_name= "Sequences", xml_doc=xml_doc, parent=node_xml,
101  values=values)
102  node_xml.appendChild(currentNode)
103  return currentNode