|
|
|
| def cmssw_exportdb_xml::_getXMLNode | ( | xml_doc | ) | [private] |
opens existing or creates a new XML object
Definition at line 98 of file cmssw_exportdb_xml.py.
| def cmssw_exportdb_xml::createNode | ( | xml_doc, | |
| node_name, | |||
values = {}, |
|||
parent = None |
|||
| ) |
Definition at line 4 of file cmssw_exportdb_xml.py.
00004 {}, parent = None): 00005 if (parent == None): 00006 parent = xml_doc 00007 #create node 00008 node = xml_doc.createElement(node_name) 00009 # assign the values 00010 for (key, value) in values.items(): 00011 node.setAttribute(key, str(value)) 00012 parent.appendChild(node) 00013 00014 return node 00015 00016
| def cmssw_exportdb_xml::export_xml | ( | release, | |
| jobID, | |||
| timelog_result, | |||
| xml_doc, | |||
metadata = None, |
|||
edmSize_result = None, |
|||
parentNode = None |
|||
| ) |
jobID is a dictionary now !
Definition at line 119 of file cmssw_exportdb_xml.py.
00120 : 00121 """ jobID is a dictionary now ! """ 00122 00123 if not parentNode: 00124 #get the root XML node 00125 parentNode = _getXMLNode(xml_doc) 00126 00127 #create jobStats node 00128 values=jobID 00129 values.update({"release": release}) 00130 if (metadata): 00131 values.update(metadata) 00132 00133 # we create a new XML element having all the statistics data for our candle, step, release 00134 jobStatsNode = createNode(node_name= "jobStats", xml_doc=xml_doc, parent=parentNode, 00135 values=values) 00136 00137 #TODO: load current module data or delete 00138 00139 (mod_timelog_result, evt_timelog_result, rss_result, vsize_result) = timelog_result 00140 00141 # modules data 00142 for (mod_name, mod_time_result) in mod_timelog_result.items(): 00143 xml_export_ModuleTimeRecord(mod_time_result, jobStatsNode, xml_doc) 00144 00145 """ #TODO: actualy we're not supposed to have any of the old statistics 00146 # clean it up if there are old stats of EventTime,RSS,VSIZE 00147 xml_delete_children(jobStatsNode, "EventTime") 00148 xml_delete_children(jobStatsNode, "EventRSS") 00149 xml_delete_children(jobStatsNode, "EventVSIZE") 00150 xml_delete_children(jobStatsNode, "EdmSize") """ 00151 00152 #events data - so far only total time per event 00153 if (evt_timelog_result): 00154 for evt_time_item in evt_timelog_result: 00155 xml_export_EventTimeRecord(evt_time_item, jobStatsNode, xml_doc) 00156 00157 # rss 00158 if (rss_result): 00159 for evt_time_item in rss_result: 00160 xml_export_EventRssRecord(evt_time_item, jobStatsNode, xml_doc) 00161 00162 # vsize 00163 if (vsize_result): 00164 for evt_time_item in vsize_result: 00165 xml_export_EventVsizeRecord(evt_time_item, jobStatsNode, xml_doc) 00166 # edmSize 00167 if (edmSize_result): 00168 for edmItem in edmSize_result: 00169 xml_export_EdmRecord(edmItem, jobStatsNode, xml_doc) 00170
| def cmssw_exportdb_xml::exportECRules | ( | xml_doc, | |
| rules | |||
| ) |
Definition at line 235 of file cmssw_exportdb_xml.py.
| def cmssw_exportdb_xml::exportRunInfo | ( | xml_doc, | |
| run_info, | |||
release = None, |
|||
print_out = False |
|||
| ) |
Definition at line 171 of file cmssw_exportdb_xml.py.
00172 : 00173 node_xml = _getXMLNode(xml_doc) 00174 #get the simple string values 00175 str_values = run_info["General"] 00176 00177 #if we have the forced release name (e.g. icludes some special tags for testing but not official release) 00178 # so TODO: probably the test_release_based_on string would be the same so we still save the (original) test release based string 00179 00180 if release: 00181 str_values["release"] = release 00182 else: 00183 str_values["release"] = str_values["test_release_based_on"] 00184 00185 runInfoNode = createNode(node_name= "RunInfo", xml_doc=xml_doc, parent=node_xml, 00186 values=str_values) 00187 #create nodes for TestResults: 00188 for (testName, result) in run_info["TestResults"].items(): 00189 #either we have one node or multiple ones (if list) 00190 if type(result) == types.ListType: 00191 for result_item in result: 00192 result_item.update({"testName": testName}) 00193 00194 #We have JOBS so FAR only for TimeSize which we represent as a list 00195 jobs = [] 00196 #we don't want jobs to be dumped as string 00197 if result_item.has_key("jobs"): 00198 jobs = result_item["jobs"] 00199 del result_item["jobs"] 00200 00201 testNode = createNode(node_name="testResult", xml_doc=xml_doc, parent=runInfoNode, values=result_item) 00202 00203 00204 for job in jobs: 00205 #print job 00206 export_xml(xml_doc = xml_doc, parentNode = testNode, **job) 00207 else: 00208 result.update({"testName": testName}) 00209 createNode(node_name="testResult", xml_doc=xml_doc, parent=runInfoNode, values=result) 00210 00211 #DO we have some unrecognized JOBS? 00212 if len(run_info['unrecognized_jobs']): 00213 unrecognizedJobsNode = createNode(node_name="Unrecognized_JOBS", xml_doc=xml_doc, parent=runInfoNode, values={}) 00214 00215 00216 for job in run_info['unrecognized_jobs']: 00217 #print job 00218 export_xml(xml_doc = xml_doc, parentNode = unrecognizedJobsNode, **job) 00219 00220 00221 #cmsSciMark 00222 cmsSciMarkNode = createNode(node_name="cmsSciMarks", xml_doc=xml_doc, parent=runInfoNode, values = {}) 00223 for csiMark in run_info["cmsSciMark"]: 00224 #print csiMark 00225 createNode(node_name="cmsSciMark", xml_doc=xml_doc, parent=cmsSciMarkNode, values=csiMark) 00226 if print_out: 00227 print xml_doc.toprettyxml(indent="\t") 00228 00229 #IgSummary 00230 IgSummaryNode = createNode(node_name="IgSummaries", xml_doc=xml_doc, parent=runInfoNode, values = {}) 00231 for iginfo in run_info["IgSummary"]: 00232 createNode(node_name="IgSummary", xml_doc=xml_doc, parent=IgSummaryNode, values=iginfo) 00233 if print_out: 00234 print xml_doc.toprettyxml(indent="\t")
| def cmssw_exportdb_xml::initXML | ( | xmldoc | ) |
opens existing or creates a new XML file
---- one of the erliest functions - quite nasty looking :)Definition at line 17 of file cmssw_exportdb_xml.py.
00018 : 00019 """ opens existing or creates a new XML file 00020 00021 ---- one of the erliest functions - quite nasty looking :)""" 00022 00023 try: 00024 node_xml = xmldoc.getElementsByTagName("xml")[0] 00025 except IndexError: 00026 #doc = minidom.Document() 00027 node_xml = createNode(xmldoc, "xml") 00028 00029 return node_xml
| def cmssw_exportdb_xml::write_xml | ( | xml_doc, | |
| remotedir, | |||
| xmlFileName | |||
| ) |
Definition at line 240 of file cmssw_exportdb_xml.py.
00241 : 00242 xml = xml_doc.toprettyxml(indent=" ") 00243 00244 # return xml as string (if requested) 00245 if (xmlFileName == ""): 00246 return xml 00247 #Adding a modification to make sure the file is written in /tmp/$USER_perfsuite_xml dir (to allow any used to harvest any workdir without permission issues. 00248 tmp_dir="/tmp/%s_perfsuite_xml"%os.getenv("USER") 00249 if not os.path.exists(tmp_dir): 00250 os.system("mkdir %s"%tmp_dir) 00251 xmlFileName=os.path.join(tmp_dir,xmlFileName) 00252 # or save that as file 00253 out = open(xmlFileName, "w") 00254 #print xml locally 00255 out.write(xml) 00256 out.close() 00257 print "Now copying %s to stage directory %s..."%(xmlFileName,remotedir) 00258 00259 #FIXME: Here we could decide to archive it on CASTOR on a dedicated directory 00260 if ":" in remotedir: #CAVEAT: since we report the timestamp as part of the xml filename we need to be careful.. 00261 (host,dir)=remotedir.split(":") 00262 #change to the directory to avoid tar replicating /tmp/$USER_perfsuite_xml/ directory structure remotely: 00263 copy_cmd='cd %s; tar cf - %s|ssh %s "cd %s;tar xf -"'%(tmp_dir,os.path.basename(xmlFileName),host,dir) 00264 else: 00265 if not os.path.exists(remotedir): 00266 os.system("mkdir %s"%remotedir) 00267 #tarpipe_cmd='cd %s;tar cf - %s|(cd %s;tar xf -)'%(tmp_dir,os.path.basename(xmlFileName),remotedir) 00268 copy_cmd='cp -pR %s/%s %s'%(tmp_dir,os.path.basename(xmlFileName),remotedir) 00269 try: 00270 print copy_cmd 00271 os.system(copy_cmd) 00272 print "Successfully copied XML report %s to stage directory %s"%(xmlFileName,remotedir) 00273 except Exception,e : 00274 print "Issues with copying XML report %s to stage directory %s!\n%s"%(xmlFileName,remotedir,str(e)) 00275 #os.system("cd -") #just in case... 00276 00277
| def cmssw_exportdb_xml::xml_delete_children | ( | nodes, | |
| child_name | |||
| ) |
Definition at line 30 of file cmssw_exportdb_xml.py.
00030 : 00031 for x in nodes.getElementsByTagName(child_name): 00032 nodes.removeChild(x) 00033 00034
| def cmssw_exportdb_xml::xml_export_EdmRecord | ( | data, | |
| curr_stat_node, | |||
| xml_doc | |||
| ) |
Definition at line 64 of file cmssw_exportdb_xml.py.
| def cmssw_exportdb_xml::xml_export_EventRssRecord | ( | evt_time_data, | |
| curr_stat_node, | |||
| xml_doc | |||
| ) |
Definition at line 54 of file cmssw_exportdb_xml.py.
| def cmssw_exportdb_xml::xml_export_EventTimeRecord | ( | evt_time_data, | |
| curr_stat_node, | |||
| xml_doc | |||
| ) |
Definition at line 48 of file cmssw_exportdb_xml.py.
| def cmssw_exportdb_xml::xml_export_EventVsizeRecord | ( | evt_time_data, | |
| curr_stat_node, | |||
| xml_doc | |||
| ) |
Definition at line 59 of file cmssw_exportdb_xml.py.
| def cmssw_exportdb_xml::xml_export_ModuleTimeRecord | ( | data_dict, | |
| curr_stat_node, | |||
| xml_doc | |||
| ) |
Definition at line 35 of file cmssw_exportdb_xml.py.
00036 : 00037 #print "Dict: "+str(data_dict) 00038 # create a module tag to be used to put the data into 00039 module_node =createNode(xml_doc, "Module", values = data_dict, parent = curr_stat_node) 00040 00041 # clean it up if there are old stats of ModuleTime, now there shouldn't be any anymore! 00042 #xml_delete_children(module_node, "ModuleTime") 00043 00044 # we create a new XML element having the statistics data 00045 moduletime_stat =createNode(xml_doc, "ModuleTime", values = data_dict["stats"], parent = module_node) 00046 00047 module_node.appendChild(moduletime_stat)
| def cmssw_exportdb_xml::xml_export_SequenceRecord | ( | data, | |
| curr_seq_node, | |||
| xml_doc | |||
| ) |
Definition at line 68 of file cmssw_exportdb_xml.py.
| def cmssw_exportdb_xml::xml_export_Sequences | ( | xml_doc, | |
| sequences, | |||
| release | |||
| ) |
Definition at line 110 of file cmssw_exportdb_xml.py.
| def cmssw_exportdb_xml::xml_init_Sequences | ( | xml_doc, | |
| release | |||
| ) |
opens existing or creates a new XML file
returns the (existing or created) unique element for statiscsDefinition at line 71 of file cmssw_exportdb_xml.py.
00072 : 00073 #TODO: this was copied from init_XML - it should not be like that - refactor 00074 """ opens existing or creates a new XML file 00075 returns the (existing or created) unique element for statiscs""" 00076 try: 00077 node_xml = xml_doc.getElementsByTagName("xml")[0] 00078 except IndexError: 00079 #doc = minidom.Document() 00080 node_xml = createNode(xml_doc, "xml") 00081 00082 nodes =node_xml.getElementsByTagName("Sequences") 00083 00084 currentNode = [node for node in nodes 00085 if (release ==node.attributes["release"].value)] 00086 #TODO: end of copied 00087 00088 if (currentNode): 00089 currentNode = currentNode[0] 00090 #TODO: this part looks nasty 00091 else: 00092 values={"release": release} 00093 # we create a new XML element having all the statistics data for our candle, step, release 00094 currentNode = createNode(node_name= "Sequences", xml_doc=xml_doc, parent=node_xml, 00095 values=values) 00096 node_xml.appendChild(currentNode) 00097 return currentNode
1.7.3