|
|
|
Functions | |
| def | getJobID_fromEdmSizeFileName |
| def | getJobID_fromFileName |
| def | getJobID_fromRootFileName |
| def | getJobID_fromTimeReportLogName |
| def | getRootFileSize |
| def | read_ConfigurationFromSimulationCandles |
| def | read_SimulationCandles |
Variables | |
| list | f_candle_and_step_inJobID = lambdacandle,step,x:x[0] |
| string | path = "/home/vidma/Desktop/CERN_code/cmssw/data/CMSSW_3_2_0_--usersteps=GEN-SIM,DIGI_lxbuild106.cern.ch_relval/relval/CMSSW_3_2_0/workGENSIMDIGI/TTbar_PU_TimeSize" |
| tuple | simCandlesRules |
| tuple | test_root_file = re.compile(".root$", re.IGNORECASE) |
| dictionary | universal_candle_step_regs = {} |
| def FileNamesHelper::getJobID_fromEdmSizeFileName | ( | logfile_name | ) |
Returns the candle and STEP out of filename:
* the candle might include one optional underscore:
>>> getJobID_fromEdmSizeFileName("E_1000_GEN,SIM_EdmSize")
('E_1000', 'GEN,SIM', '', '')
* otherwise after candle we have two underscores:
>>> getJobID_fromEdmSizeFileName("TTBAR__RAW2DIGI,RECO_EdmSize")
('TTBAR', 'RAW2DIGI,RECO', '', '')
* and lastly we have the PILEUP possibility:
>>> getJobID_fromEdmSizeFileName("TTBAR__GEN,SIM_PILEUP_EdmSize")
('TTBAR', 'GEN,SIM', 'PILEUP', '')
Definition at line 184 of file FileNamesHelper.py.
00185 : 00186 """ 00187 Returns the candle and STEP out of filename: 00188 00189 * the candle might include one optional underscore: 00190 >>> getJobID_fromEdmSizeFileName("E_1000_GEN,SIM_EdmSize") 00191 ('E_1000', 'GEN,SIM', '', '') 00192 00193 * otherwise after candle we have two underscores: 00194 >>> getJobID_fromEdmSizeFileName("TTBAR__RAW2DIGI,RECO_EdmSize") 00195 ('TTBAR', 'RAW2DIGI,RECO', '', '') 00196 00197 * and lastly we have the PILEUP possibility: 00198 >>> getJobID_fromEdmSizeFileName("TTBAR__GEN,SIM_PILEUP_EdmSize") 00199 ('TTBAR', 'GEN,SIM', 'PILEUP', '') 00200 """ 00201 return getJobID_fromFileName(logfile_name, "_EdmSize")
| def FileNamesHelper::getJobID_fromFileName | ( | logfile_name, | |
| suffix, | |||
givenPath = "" |
|||
| ) |
Returns the JobID (candle, step, pileup_type, conditions, event_content) out of filename
-- if no pile up returns empty string for pileup type
* the candle might include one optional underscore:
>>> getJobID_fromFileName("PI-_1000_GEN,SIM.root", "\.root")
('PI-_1000', 'GEN,SIM', '', '')
* otherwise after candle we have two underscores:
>>> getJobID_fromFileName("MINBIAS__GEN,FASTSIM.root", "\.root")
('MINBIAS', 'GEN,FASTSIM', '', '')
* and lastly we have the PILEUP possibility:
>>> getJobID_fromFileName("TTBAR__DIGI_PILEUP.root", "\.root")
('TTBAR', 'DIGI', 'PILEUP', '')
Definition at line 92 of file FileNamesHelper.py.
00093 : 00094 #TODO: join together with the one from parseTimingReport.py 00095 """ 00096 Returns the JobID (candle, step, pileup_type, conditions, event_content) out of filename 00097 -- if no pile up returns empty string for pileup type 00098 00099 * the candle might include one optional underscore: 00100 >>> getJobID_fromFileName("PI-_1000_GEN,SIM.root", "\.root") 00101 ('PI-_1000', 'GEN,SIM', '', '') 00102 00103 * otherwise after candle we have two underscores: 00104 >>> getJobID_fromFileName("MINBIAS__GEN,FASTSIM.root", "\.root") 00105 ('MINBIAS', 'GEN,FASTSIM', '', '') 00106 00107 * and lastly we have the PILEUP possibility: 00108 >>> getJobID_fromFileName("TTBAR__DIGI_PILEUP.root", "\.root") 00109 ('TTBAR', 'DIGI', 'PILEUP', '') 00110 """ 00111 import os 00112 00113 # get the actual filename (no path) 00114 (path, filename) = os.path.split(logfile_name) 00115 if givenPath: 00116 path = givenPath 00117 00118 if not universal_candle_step_regs.has_key(suffix): 00119 #create and cache a regexp 00120 universal_candle_step_regs[suffix] = re.compile( \ 00121 r""" 00122 #candle1_[opt:candle2]_ 00123 ^([^_]+_[^_]*)_ 00124 00125 # step 00126 ([^_]+)(_PILEUP)?%s$ 00127 """ % suffix , re.VERBOSE) 00128 00129 00130 00131 #print logfile_name 00132 result = universal_candle_step_regs[suffix].search(filename) 00133 if result: 00134 #print result.groups() 00135 #print "result: %s" % str(result.groups()) 00136 candle = result.groups()[0] 00137 step = result.groups()[1] 00138 is_pileup = result.groups()[2] 00139 if is_pileup: 00140 is_pileup = "PILEUP" 00141 else: 00142 is_pileup = "" 00143 00144 """ if we had the candle without underscore inside (like TTBAR but not E_1000) 00145 on the end of result and underscore which needs to be removed """ 00146 00147 if (candle[-1] == '_'): 00148 candle = candle[0:-1] 00149 00150 """ try to fetch the conditions and real pileup type if the SimulationCandles.txt is existing """ 00151 conditions = '' 00152 event_content = '' 00153 try: 00154 conf = read_ConfigurationFromSimulationCandles(path = path, step = step, is_pileup= is_pileup) 00155 if conf: 00156 is_pileup = conf["pileup_type"] 00157 conditions = conf["conditions"] 00158 event_content = conf["event_content"] 00159 except OSError, e: 00160 pass 00161 00162 return (candle, step, is_pileup, conditions, event_content) 00163 else: 00164 return (None, None, None, None, None) 00165
| def FileNamesHelper::getJobID_fromRootFileName | ( | logfile_name | ) |
Returns the candle and STEP out of filename:
* the candle might include one optional underscore:
>>> getJobID_fromRootFileName("PI-_1000_GEN,SIM.root")
('PI-_1000', 'GEN,SIM', '', '')
* otherwise after candle we have two underscores:
>>> getJobID_fromRootFileName("MINBIAS__GEN,FASTSIM.root")
('MINBIAS', 'GEN,FASTSIM', '', '')
* and lastly we have the PILEUP possibility:
>>> getJobID_fromRootFileName("TTBAR__DIGI_PILEUP.root")
('TTBAR', 'DIGI', 'PILEUP', '')
Definition at line 166 of file FileNamesHelper.py.
00167 : 00168 """ 00169 Returns the candle and STEP out of filename: 00170 00171 * the candle might include one optional underscore: 00172 >>> getJobID_fromRootFileName("PI-_1000_GEN,SIM.root") 00173 ('PI-_1000', 'GEN,SIM', '', '') 00174 00175 * otherwise after candle we have two underscores: 00176 >>> getJobID_fromRootFileName("MINBIAS__GEN,FASTSIM.root") 00177 ('MINBIAS', 'GEN,FASTSIM', '', '') 00178 00179 * and lastly we have the PILEUP possibility: 00180 >>> getJobID_fromRootFileName("TTBAR__DIGI_PILEUP.root") 00181 ('TTBAR', 'DIGI', 'PILEUP', '') 00182 """ 00183 return getJobID_fromFileName(logfile_name, "\\.root")
| def FileNamesHelper::getJobID_fromTimeReportLogName | ( | logfile_name | ) |
Returns the candle and STEP out of filename:
* the candle might include one optional underscore:
>>> getJobID_fromTimeReportLogName("E_1000_GEN,SIM_TimingReport.log")
('E_1000', 'GEN,SIM', '', '')
* otherwise after candle we have two underscores:
>>> getJobID_fromTimeReportLogName("test_data/TTBAR__RAW2DIGI,RECO_TimingReport.log")
('TTBAR', 'RAW2DIGI,RECO', '', '')
* and lastly we have the PILEUP possibility:
>>> getJobID_fromTimeReportLogName("TTBAR__DIGI_PILEUP_TimingReport.log")
('TTBAR', 'DIGI', 'PILEUP', '')
Definition at line 202 of file FileNamesHelper.py.
00203 : 00204 """ 00205 Returns the candle and STEP out of filename: 00206 00207 * the candle might include one optional underscore: 00208 >>> getJobID_fromTimeReportLogName("E_1000_GEN,SIM_TimingReport.log") 00209 ('E_1000', 'GEN,SIM', '', '') 00210 00211 * otherwise after candle we have two underscores: 00212 >>> getJobID_fromTimeReportLogName("test_data/TTBAR__RAW2DIGI,RECO_TimingReport.log") 00213 ('TTBAR', 'RAW2DIGI,RECO', '', '') 00214 00215 * and lastly we have the PILEUP possibility: 00216 >>> getJobID_fromTimeReportLogName("TTBAR__DIGI_PILEUP_TimingReport.log") 00217 ('TTBAR', 'DIGI', 'PILEUP', '') 00218 """ 00219 return getJobID_fromFileName(logfile_name, "_TimingReport.log") 00220 00221 """ Get the root file size for the candle, step in current dir """
| def FileNamesHelper::getRootFileSize | ( | path, | |
| candle, | |||
| step | |||
| ) |
Definition at line 222 of file FileNamesHelper.py.
00223 : 00224 files = os.listdir(path) 00225 root_files = [os.path.join(path, f) for f in files 00226 if test_root_file.search(f) 00227 and os.path.isfile(os.path.join(path, f)) ] 00228 00229 """ get the size of file if it is the root file for current candle and step """ 00230 try: 00231 size = [os.stat(f).st_size for f in root_files 00232 if f_candle_and_step_inJobID(candle, step, getJobID_fromRootFileName(f))][0] 00233 except Exception, e: 00234 print e 00235 return 0 00236 return size
| def FileNamesHelper::read_ConfigurationFromSimulationCandles | ( | path, | |
| step, | |||
| is_pileup | |||
| ) |
Definition at line 45 of file FileNamesHelper.py.
00046 : 00047 # Here we parse SimulationCandles_<version: e.g. CMSSW_3_2_0>.txt which contains 00048 # release:TODO, release_base [path] - we can put it to release [but it's of different granularity] 00049 # how to reproduce stuff: TODO 00050 00051 try: 00052 """ get the acual file """ 00053 SimulationCandles_file = [os.path.join(path, f) for f in os.listdir(path) 00054 if os.path.isfile(os.path.join(path, f)) and f.startswith("SimulationCandles_")][0] 00055 except IndexError: 00056 return None 00057 00058 """ read and parse it; format: #Version : CMSSW_3_2_0 """ 00059 f = open(SimulationCandles_file, 'r') 00060 00061 lines = [s.strip() for s in f.readlines()] 00062 f.close() 00063 00064 00065 00066 """ we call a shared helper to parse the file """ 00067 00068 for line in lines: 00069 #print line 00070 #print simCandlesRules[2][1].match(line) and simCandlesRules[2][1].match(line).groups() or "" 00071 00072 info, missing_fields = parsingRulesHelper.rulesParser(simCandlesRules, [line], compileRules = False) 00073 #Massaging the info dictionary conditions entry to allow for new cmsDriver.py --conditions option: 00074 if 'auto:' in info['conditions']: 00075 from Configuration.PyReleaseValidation.autoCond import autoCond 00076 info['conditions'] = autoCond[ info['conditions'].split(':')[1] ].split("::")[0] 00077 else: 00078 if 'FrontierConditions_GlobalTag' in info['conditions']: 00079 info['conditions']=info['conditions'].split(",")[1] 00080 #print (info, missing_fields) 00081 #if we successfully parsed the line of simulation candles: 00082 if not missing_fields: 00083 #we have to match only step and 00084 if info["step"].strip() == step.strip() and ((not is_pileup and not info["pileup_type"]) or (is_pileup and info["pileup_type"])): 00085 # if it's pile up or not: 00086 #print "Info for <<%s, %s>>: %s" % (str(step), str(is_pileup), str(info)) 00087 return info 00088 00089 00090 00091
| def FileNamesHelper::read_SimulationCandles | ( | path | ) |
get the acual file
Definition at line 237 of file FileNamesHelper.py.
00238 : 00239 # Here we parse SimulationCandles_<version: e.g. CMSSW_3_2_0>.txt which contains 00240 # release:TODO, release_base [path] - we can put it to release [but it's of different granularity] 00241 # how to reproduce stuff: TODO 00242 00243 """ get the acual file """ 00244 SimulationCandles_file = [os.path.join(path, f) for f in os.listdir(path) 00245 if os.path.isfile(os.path.join(path, f)) and f.startswith("SimulationCandles_")][0] 00246 00247 """ read and parse it; format: #Version : CMSSW_3_2_0 """ 00248 f = open(SimulationCandles_file, 'r') 00249 lines = f.readlines() 00250 f.close() 00251 00252 release_version =[[a.strip() for a in line.split(":")] for line in lines if line.startswith("#Version")][0][1] 00253 return release_version 00254
| list FileNamesHelper::f_candle_and_step_inJobID = lambdacandle,step,x:x[0] |
Definition at line 6 of file FileNamesHelper.py.
| string FileNamesHelper::path = "/home/vidma/Desktop/CERN_code/cmssw/data/CMSSW_3_2_0_--usersteps=GEN-SIM,DIGI_lxbuild106.cern.ch_relval/relval/CMSSW_3_2_0/workGENSIMDIGI/TTbar_PU_TimeSize" |
Definition at line 258 of file FileNamesHelper.py.
00001 ( 00002 00003 #e.g.: --conditions FrontierConditions_GlobalTag,MC_31X_V4::All --eventcontent RECOSIM 00004 (("cms_driver_options", ), r"""^cmsDriver.py(.+)$"""), 00005 #Changing the following to allow for new cmsDriver.py --conditions option (that can optionally drop the FrontierConditions_GlobalTag,) 00006 (("", "conditions", ""), r"""^cmsDriver.py(.*)--conditions ([^\s]+)(.*)$""", "req"), 00007 (("", "pileup_type", ""), r"""^cmsDriver.py(.*)--pileup=([^\s]+)(.*)$"""), 00008 (("", "step", ""), r"""^cmsDriver.py(.*)--step=([^\s]+)(.*)$""", "req"), 00009 #not shure if event content is required 00010 (("", "event_content", ""), r"""^cmsDriver.py(.*)--eventcontent ([^\s]+)(.*)$""", "req"), 00011 (("", "num_events", ""), r"""^cmsDriver.py(.*)-n ([^\s]+)(.*)$""", "req"), 00012 00013 #TODO: after changeing the splitter to "taskset -c ..." this is no longer included into the part of correct job 00014 #(("input_user_root_file", ), r"""^For these tests will use user input file (.+)$"""), 00015 )
Definition at line 28 of file FileNamesHelper.py.
| tuple FileNamesHelper::test_root_file = re.compile(".root$", re.IGNORECASE) |
Definition at line 19 of file FileNamesHelper.py.
| dictionary FileNamesHelper::universal_candle_step_regs = {} |
Definition at line 18 of file FileNamesHelper.py.
1.7.3