|
|
|
Functions | |
| def | getJobID_fromEdmSizeFileName |
| def | getJobID_fromFileName |
| def | getJobID_fromIgProfLogName |
| def | getJobID_fromMemcheckLogName |
| def | getJobID_fromRootFileName |
| def | getJobID_fromTimeReportLogName |
| def | getRootFileSize |
| def | read_ConfigurationFromSimulationCandles |
| def | read_SimulationCandles |
Variables | |
| list | f_candle_and_step_inJobID = lambdacandle,step,x:x[0] |
| string | path = "/home/vidma/Desktop/CERN_code/cmssw/data/CMSSW_3_2_0_--usersteps=GEN-SIM,DIGI_lxbuild106.cern.ch_relval/relval/CMSSW_3_2_0/workGENSIMDIGI/TTbar_PU_TimeSize" |
| tuple | simCandlesRules |
| tuple | test_root_file = re.compile(".root$", re.IGNORECASE) |
| dictionary | universal_candle_step_regs = {} |
| def FileNamesHelper::getJobID_fromEdmSizeFileName | ( | logfile_name | ) |
Returns the candle and STEP out of filename:
* the candle might include one optional underscore:
>>> getJobID_fromEdmSizeFileName("E_1000_GEN,SIM_EdmSize")
('E_1000', 'GEN,SIM', '', '')
* otherwise after candle we have two underscores:
>>> getJobID_fromEdmSizeFileName("TTBAR__RAW2DIGI,RECO_EdmSize")
('TTBAR', 'RAW2DIGI,RECO', '', '')
* and lastly we have the PILEUP possibility:
>>> getJobID_fromEdmSizeFileName("TTBAR__GEN,SIM_PILEUP_EdmSize")
('TTBAR', 'GEN,SIM', 'PILEUP', '')
Definition at line 185 of file FileNamesHelper.py.
00186 : 00187 """ 00188 Returns the candle and STEP out of filename: 00189 00190 * the candle might include one optional underscore: 00191 >>> getJobID_fromEdmSizeFileName("E_1000_GEN,SIM_EdmSize") 00192 ('E_1000', 'GEN,SIM', '', '') 00193 00194 * otherwise after candle we have two underscores: 00195 >>> getJobID_fromEdmSizeFileName("TTBAR__RAW2DIGI,RECO_EdmSize") 00196 ('TTBAR', 'RAW2DIGI,RECO', '', '') 00197 00198 * and lastly we have the PILEUP possibility: 00199 >>> getJobID_fromEdmSizeFileName("TTBAR__GEN,SIM_PILEUP_EdmSize") 00200 ('TTBAR', 'GEN,SIM', 'PILEUP', '') 00201 """ 00202 return getJobID_fromFileName(logfile_name, "_EdmSize")
| def FileNamesHelper::getJobID_fromFileName | ( | logfile_name, | |
| suffix, | |||
givenPath = "" |
|||
| ) |
Returns the JobID (candle, step, pileup_type, conditions, event_content) out of filename
-- if no pile up returns empty string for pileup type
* the candle might include one optional underscore:
>>> getJobID_fromFileName("PI-_1000_GEN,SIM.root", "\.root")
('PI-_1000', 'GEN,SIM', '', '')
* otherwise after candle we have two underscores:
>>> getJobID_fromFileName("MINBIAS__GEN,FASTSIM.root", "\.root")
('MINBIAS', 'GEN,FASTSIM', '', '')
* and lastly we have the PILEUP possibility:
>>> getJobID_fromFileName("TTBAR__DIGI_PILEUP.root", "\.root")
('TTBAR', 'DIGI', 'PILEUP', '')
Definition at line 93 of file FileNamesHelper.py.
00094 : 00095 #TODO: join together with the one from parseTimingReport.py 00096 """ 00097 Returns the JobID (candle, step, pileup_type, conditions, event_content) out of filename 00098 -- if no pile up returns empty string for pileup type 00099 00100 * the candle might include one optional underscore: 00101 >>> getJobID_fromFileName("PI-_1000_GEN,SIM.root", "\.root") 00102 ('PI-_1000', 'GEN,SIM', '', '') 00103 00104 * otherwise after candle we have two underscores: 00105 >>> getJobID_fromFileName("MINBIAS__GEN,FASTSIM.root", "\.root") 00106 ('MINBIAS', 'GEN,FASTSIM', '', '') 00107 00108 * and lastly we have the PILEUP possibility: 00109 >>> getJobID_fromFileName("TTBAR__DIGI_PILEUP.root", "\.root") 00110 ('TTBAR', 'DIGI', 'PILEUP', '') 00111 """ 00112 import os 00113 00114 # get the actual filename (no path 00115 (path, filename) = os.path.split(logfile_name) 00116 if givenPath: 00117 path = givenPath 00118 00119 if not universal_candle_step_regs.has_key(suffix): 00120 #create and cache a regexp 00121 universal_candle_step_regs[suffix] = re.compile( \ 00122 r""" 00123 #candle1_[opt:candle2]_ 00124 ^([^_]+_[^_]*)_ 00125 00126 # step 00127 ([^_]+)(_PILEUP)?%s$ 00128 """ % suffix , re.VERBOSE) 00129 00130 00131 00132 #print logfile_name 00133 result = universal_candle_step_regs[suffix].search(filename) 00134 if result: 00135 #print result.groups() 00136 #print "result: %s" % str(result.groups()) 00137 candle = result.groups()[0] 00138 step = result.groups()[1].replace('-', ',') 00139 is_pileup = result.groups()[2] 00140 if is_pileup: 00141 is_pileup = "PILEUP" 00142 else: 00143 is_pileup = "" 00144 00145 """ if we had the candle without underscore inside (like TTBAR but not E_1000) 00146 on the end of result and underscore which needs to be removed """ 00147 00148 if (candle[-1] == '_'): 00149 candle = candle[0:-1] 00150 00151 """ try to fetch the conditions and real pileup type if the SimulationCandles.txt is existing """ 00152 conditions = '' 00153 event_content = '' 00154 try: 00155 conf = read_ConfigurationFromSimulationCandles(path = path, step = step, is_pileup= is_pileup) 00156 if conf: 00157 is_pileup = conf["pileup_type"] 00158 conditions = conf["conditions"] 00159 event_content = conf["event_content"] 00160 except OSError, e: 00161 pass 00162 00163 return (candle, step, is_pileup, conditions, event_content) 00164 else: 00165 return (None, None, None, None, None) 00166
| def FileNamesHelper::getJobID_fromIgProfLogName | ( | logfile_name | ) |
Returns the candle and STEP out of .sql3 filename: everything is given, just have to split it... like: TTbar___GEN,FASTSIM___LowLumiPileUp___MC_37Y_V5___RAWSIM___MEM_LIVE___1.sql3 and correct the conditions!
Definition at line 235 of file FileNamesHelper.py.
00236 : 00237 """ 00238 Returns the candle and STEP out of .sql3 filename: 00239 00240 everything is given, just have to split it... 00241 like: 00242 TTbar___GEN,FASTSIM___LowLumiPileUp___MC_37Y_V5___RAWSIM___MEM_LIVE___1.sql3 00243 and correct the conditions! 00244 00245 """ 00246 00247 (path, filename) = os.path.split(logfile_name) 00248 00249 params = filename.split("___") 00250 candle = params[0].upper() 00251 step = params[1] 00252 pileup_type = params[2] 00253 if pileup_type == "NOPILEUP": 00254 pileup_type = "" 00255 elif pileup_type == "LowLumiPileUp": 00256 pileup_type = "PILEUP" 00257 #conditions = params[3] + "::All" 00258 #event_content = params[4] 00259 00260 #get the conditions from the SimulationCandles!! 00261 conf = read_ConfigurationFromSimulationCandles(path = path, step = step, is_pileup= pileup_type) 00262 if conf: 00263 is_pileup = conf["pileup_type"] 00264 conditions = conf["conditions"] 00265 event_content = conf["event_content"] 00266 return (candle, step, is_pileup, conditions, event_content) 00267 else: 00268 return (None, None, None, None, None) 00269 """ Get the root file size for the candle, step in current dir """
| def FileNamesHelper::getJobID_fromMemcheckLogName | ( | logfile_name | ) |
Returns the candle and STEP out of filename:
* otherwise after candle we have two underscores:
>>> getJobID_fromTimeReportLogName("test_data/TTBAR__RAW2DIGI,RECO_memcheck_vlgd.xml")
('TTBAR', 'RAW2DIGI,RECO', '', '')
* and lastly we have the PILEUP possibility:
>>> getJobID_fromTimeReportLogName("TTBAR__DIGI_PILEUP_memcheck_vlgd.xml")
('TTBAR', 'DIGI', 'PILEUP', '')
Definition at line 221 of file FileNamesHelper.py.
00222 : 00223 """ 00224 Returns the candle and STEP out of filename: 00225 00226 * otherwise after candle we have two underscores: 00227 >>> getJobID_fromTimeReportLogName("test_data/TTBAR__RAW2DIGI,RECO_memcheck_vlgd.xml") 00228 ('TTBAR', 'RAW2DIGI,RECO', '', '') 00229 00230 * and lastly we have the PILEUP possibility: 00231 >>> getJobID_fromTimeReportLogName("TTBAR__DIGI_PILEUP_memcheck_vlgd.xml") 00232 ('TTBAR', 'DIGI', 'PILEUP', '') 00233 """ 00234 return getJobID_fromFileName(logfile_name, "_memcheck_vlgd.xml")
| def FileNamesHelper::getJobID_fromRootFileName | ( | logfile_name | ) |
Returns the candle and STEP out of filename:
* the candle might include one optional underscore:
>>> getJobID_fromRootFileName("PI-_1000_GEN,SIM.root")
('PI-_1000', 'GEN,SIM', '', '')
* otherwise after candle we have two underscores:
>>> getJobID_fromRootFileName("MINBIAS__GEN,FASTSIM.root")
('MINBIAS', 'GEN,FASTSIM', '', '')
* and lastly we have the PILEUP possibility:
>>> getJobID_fromRootFileName("TTBAR__DIGI_PILEUP.root")
('TTBAR', 'DIGI', 'PILEUP', '')
Definition at line 167 of file FileNamesHelper.py.
00168 : 00169 """ 00170 Returns the candle and STEP out of filename: 00171 00172 * the candle might include one optional underscore: 00173 >>> getJobID_fromRootFileName("PI-_1000_GEN,SIM.root") 00174 ('PI-_1000', 'GEN,SIM', '', '') 00175 00176 * otherwise after candle we have two underscores: 00177 >>> getJobID_fromRootFileName("MINBIAS__GEN,FASTSIM.root") 00178 ('MINBIAS', 'GEN,FASTSIM', '', '') 00179 00180 * and lastly we have the PILEUP possibility: 00181 >>> getJobID_fromRootFileName("TTBAR__DIGI_PILEUP.root") 00182 ('TTBAR', 'DIGI', 'PILEUP', '') 00183 """ 00184 return getJobID_fromFileName(logfile_name, "\\.root")
| def FileNamesHelper::getJobID_fromTimeReportLogName | ( | logfile_name | ) |
Returns the candle and STEP out of filename:
* the candle might include one optional underscore:
>>> getJobID_fromTimeReportLogName("E_1000_GEN,SIM_TimingReport.log")
('E_1000', 'GEN,SIM', '', '')
* otherwise after candle we have two underscores:
>>> getJobID_fromTimeReportLogName("test_data/TTBAR__RAW2DIGI,RECO_TimingReport.log")
('TTBAR', 'RAW2DIGI,RECO', '', '')
* and lastly we have the PILEUP possibility:
>>> getJobID_fromTimeReportLogName("TTBAR__DIGI_PILEUP_TimingReport.log")
('TTBAR', 'DIGI', 'PILEUP', '')
Definition at line 203 of file FileNamesHelper.py.
00204 : 00205 """ 00206 Returns the candle and STEP out of filename: 00207 00208 * the candle might include one optional underscore: 00209 >>> getJobID_fromTimeReportLogName("E_1000_GEN,SIM_TimingReport.log") 00210 ('E_1000', 'GEN,SIM', '', '') 00211 00212 * otherwise after candle we have two underscores: 00213 >>> getJobID_fromTimeReportLogName("test_data/TTBAR__RAW2DIGI,RECO_TimingReport.log") 00214 ('TTBAR', 'RAW2DIGI,RECO', '', '') 00215 00216 * and lastly we have the PILEUP possibility: 00217 >>> getJobID_fromTimeReportLogName("TTBAR__DIGI_PILEUP_TimingReport.log") 00218 ('TTBAR', 'DIGI', 'PILEUP', '') 00219 """ 00220 return getJobID_fromFileName(logfile_name, "_TimingReport.log")
| def FileNamesHelper::getRootFileSize | ( | path, | |
| candle, | |||
| step | |||
| ) |
Definition at line 270 of file FileNamesHelper.py.
00271 : 00272 files = os.listdir(path) 00273 root_files = [os.path.join(path, f) for f in files 00274 if test_root_file.search(f) 00275 and os.path.isfile(os.path.join(path, f)) ] 00276 00277 """ get the size of file if it is the root file for current candle and step """ 00278 try: 00279 size = [os.stat(f).st_size for f in root_files 00280 if f_candle_and_step_inJobID(candle, step, getJobID_fromRootFileName(f))][0] 00281 except Exception, e: 00282 print e 00283 return 0 00284 return size
| def FileNamesHelper::read_ConfigurationFromSimulationCandles | ( | path, | |
| step, | |||
| is_pileup | |||
| ) |
Definition at line 45 of file FileNamesHelper.py.
00046 : 00047 # Here we parse SimulationCandles_<version: e.g. CMSSW_3_2_0>.txt which contains 00048 # release:TODO, release_base [path] - we can put it to release [but it's of different granularity] 00049 # how to reproduce stuff: TODO 00050 00051 try: 00052 """ get the acual file """ 00053 SimulationCandles_file = [os.path.join(path, f) for f in os.listdir(path) 00054 if os.path.isfile(os.path.join(path, f)) and f.startswith("SimulationCandles_")][0] 00055 except IndexError: 00056 return None 00057 00058 """ read and parse it; format: #Version : CMSSW_3_2_0 """ 00059 f = open(SimulationCandles_file, 'r') 00060 00061 lines = [s.strip() for s in f.readlines()] 00062 f.close() 00063 00064 00065 00066 """ we call a shared helper to parse the file """ 00067 00068 for line in lines: 00069 #print line 00070 #print simCandlesRules[2][1].match(line) and simCandlesRules[2][1].match(line).groups() or "" 00071 00072 info, missing_fields = parsingRulesHelper.rulesParser(simCandlesRules, [line], compileRules = False) 00073 #print info 00074 #Massaging the info dictionary conditions entry to allow for new cmsDriver.py --conditions option: 00075 if 'auto:' in info['conditions']: 00076 from Configuration.AlCa.autoCond import autoCond 00077 info['conditions'] = autoCond[ info['conditions'].split(':')[1] ].split("::")[0] 00078 else: 00079 if 'FrontierConditions_GlobalTag' in info['conditions']: 00080 info['conditions']=info['conditions'].split(",")[1] 00081 #print (info, missing_fields) 00082 #if we successfully parsed the line of simulation candles: 00083 if not missing_fields: 00084 #we have to match only step and 00085 if info["step"].strip() == step.strip() and ((not is_pileup and not info["pileup_type"]) or (is_pileup and info["pileup_type"])): 00086 # if it's pile up or not: 00087 #print "Info for <<%s, %s>>: %s" % (str(step), str(is_pileup), str(info)) 00088 return info 00089 00090 00091 00092
| def FileNamesHelper::read_SimulationCandles | ( | path | ) |
get the acual file
Definition at line 285 of file FileNamesHelper.py.
00286 : 00287 # Here we parse SimulationCandles_<version: e.g. CMSSW_3_2_0>.txt which contains 00288 # release:TODO, release_base [path] - we can put it to release [but it's of different granularity] 00289 # how to reproduce stuff: TODO 00290 00291 """ get the acual file """ 00292 SimulationCandles_file = [os.path.join(path, f) for f in os.listdir(path) 00293 if os.path.isfile(os.path.join(path, f)) and f.startswith("SimulationCandles_")][0] 00294 00295 """ read and parse it; format: #Version : CMSSW_3_2_0 """ 00296 f = open(SimulationCandles_file, 'r') 00297 lines = f.readlines() 00298 f.close() 00299 00300 release_version =[[a.strip() for a in line.split(":")] for line in lines if line.startswith("#Version")][0][1] 00301 return release_version 00302
| list FileNamesHelper::f_candle_and_step_inJobID = lambdacandle,step,x:x[0] |
Definition at line 6 of file FileNamesHelper.py.
| string FileNamesHelper::path = "/home/vidma/Desktop/CERN_code/cmssw/data/CMSSW_3_2_0_--usersteps=GEN-SIM,DIGI_lxbuild106.cern.ch_relval/relval/CMSSW_3_2_0/workGENSIMDIGI/TTbar_PU_TimeSize" |
Definition at line 306 of file FileNamesHelper.py.
00001 ( 00002 00003 #e.g.: --conditions FrontierConditions_GlobalTag,MC_31X_V4::All --eventcontent RECOSIM 00004 (("cms_driver_options", ), r"""^cmsDriver.py(.+)$"""), 00005 #Changing the following to allow for new cmsDriver.py --conditions option (that can optionally drop the FrontierConditions_GlobalTag,) 00006 (("", "conditions", ""), r"""^cmsDriver.py(.*)--conditions ([^\s]+)(.*)$""", "req"), 00007 (("", "pileup_type", ""), r"""^cmsDriver.py(.*)--pileup=([^\s]+)(.*)$"""), 00008 (("", "step", ""), r"""^cmsDriver.py(.*)--step=([^\s]+)(.*)$""", "req"), 00009 #not shure if event content is required 00010 (("", "event_content", ""), r"""^cmsDriver.py(.*)--eventcontent ([^\s]+)(.*)$""", "req"), 00011 (("", "num_events", ""), r"""^cmsDriver.py(.*)-n ([^\s]+)(.*)$""", "req"), 00012 00013 #TODO: after changeing the splitter to "taskset -c ..." this is no longer included into the part of correct job 00014 #(("input_user_root_file", ), r"""^For these tests will use user input file (.+)$"""), 00015 )
Definition at line 28 of file FileNamesHelper.py.
| tuple FileNamesHelper::test_root_file = re.compile(".root$", re.IGNORECASE) |
Definition at line 19 of file FileNamesHelper.py.
| dictionary FileNamesHelper::universal_candle_step_regs = {} |
Definition at line 18 of file FileNamesHelper.py.
1.7.3