CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
FileNamesHelper Namespace Reference

Functions

def getJobID_fromEdmSizeFileName
 
def getJobID_fromFileName
 
def getJobID_fromRootFileName
 
def getJobID_fromTimeReportLogName
 
def getRootFileSize
 
def read_ConfigurationFromSimulationCandles
 
def read_SimulationCandles
 

Variables

list f_candle_and_step_inJobID = lambdacandle,step,x:x[0]
 
string path = "/home/vidma/Desktop/CERN_code/cmssw/data/CMSSW_3_2_0_--usersteps=GEN-SIM,DIGI_lxbuild106.cern.ch_relval/relval/CMSSW_3_2_0/workGENSIMDIGI/TTbar_PU_TimeSize"
 
tuple simCandlesRules
 
tuple test_root_file = re.compile(".root$", re.IGNORECASE)
 
dictionary universal_candle_step_regs = {}
 

Function Documentation

def FileNamesHelper.getJobID_fromEdmSizeFileName (   logfile_name)
Returns the candle and STEP out of filename:

* the candle might include one optional underscore:
>>> getJobID_fromEdmSizeFileName("E_1000_GEN,SIM_EdmSize")
('E_1000', 'GEN,SIM', '', '')

* otherwise after candle we have two underscores:
>>> getJobID_fromEdmSizeFileName("TTBAR__RAW2DIGI,RECO_EdmSize")
('TTBAR', 'RAW2DIGI,RECO', '', '')

* and lastly we have the PILEUP possibility:
>>> getJobID_fromEdmSizeFileName("TTBAR__GEN,SIM_PILEUP_EdmSize")
('TTBAR', 'GEN,SIM', 'PILEUP', '')

Definition at line 184 of file FileNamesHelper.py.

References getJobID_fromFileName().

Referenced by parserEdmSize.getEdmReport().

185 def getJobID_fromEdmSizeFileName(logfile_name):
186  """
187  Returns the candle and STEP out of filename:
188 
189  * the candle might include one optional underscore:
190  >>> getJobID_fromEdmSizeFileName("E_1000_GEN,SIM_EdmSize")
191  ('E_1000', 'GEN,SIM', '', '')
192 
193  * otherwise after candle we have two underscores:
194  >>> getJobID_fromEdmSizeFileName("TTBAR__RAW2DIGI,RECO_EdmSize")
195  ('TTBAR', 'RAW2DIGI,RECO', '', '')
196 
197  * and lastly we have the PILEUP possibility:
198  >>> getJobID_fromEdmSizeFileName("TTBAR__GEN,SIM_PILEUP_EdmSize")
199  ('TTBAR', 'GEN,SIM', 'PILEUP', '')
200  """
201  return getJobID_fromFileName(logfile_name, "_EdmSize")
def getJobID_fromEdmSizeFileName
def FileNamesHelper.getJobID_fromFileName (   logfile_name,
  suffix,
  givenPath = "" 
)
Returns the JobID (candle, step, pileup_type, conditions, event_content) out of filename
-- if no pile up returns empty string for pileup type

* the candle might include one optional underscore:
>>> getJobID_fromFileName("PI-_1000_GEN,SIM.root", "\.root")
('PI-_1000', 'GEN,SIM', '', '')

* otherwise after candle we have two underscores:
>>> getJobID_fromFileName("MINBIAS__GEN,FASTSIM.root", "\.root")
('MINBIAS', 'GEN,FASTSIM', '', '')

* and lastly we have the PILEUP possibility:
>>> getJobID_fromFileName("TTBAR__DIGI_PILEUP.root", "\.root")
('TTBAR', 'DIGI', 'PILEUP', '')

Definition at line 92 of file FileNamesHelper.py.

References read_ConfigurationFromSimulationCandles(), and electronDbsDiscovery.search().

Referenced by getJobID_fromEdmSizeFileName(), getJobID_fromRootFileName(), and getJobID_fromTimeReportLogName().

92 
93 def getJobID_fromFileName(logfile_name, suffix, givenPath =""):
94  #TODO: join together with the one from parseTimingReport.py
95  """
96  Returns the JobID (candle, step, pileup_type, conditions, event_content) out of filename
97  -- if no pile up returns empty string for pileup type
98 
99  * the candle might include one optional underscore:
100  >>> getJobID_fromFileName("PI-_1000_GEN,SIM.root", "\.root")
101  ('PI-_1000', 'GEN,SIM', '', '')
102 
103  * otherwise after candle we have two underscores:
104  >>> getJobID_fromFileName("MINBIAS__GEN,FASTSIM.root", "\.root")
105  ('MINBIAS', 'GEN,FASTSIM', '', '')
106 
107  * and lastly we have the PILEUP possibility:
108  >>> getJobID_fromFileName("TTBAR__DIGI_PILEUP.root", "\.root")
109  ('TTBAR', 'DIGI', 'PILEUP', '')
110  """
111  import os
112 
113  # get the actual filename (no path)
114  (path, filename) = os.path.split(logfile_name)
115  if givenPath:
116  path = givenPath
117 
118  if not universal_candle_step_regs.has_key(suffix):
119  #create and cache a regexp
120  universal_candle_step_regs[suffix] = re.compile( \
121  r"""
122  #candle1_[opt:candle2]_
123  ^([^_]+_[^_]*)_
124 
125  # step
126  ([^_]+)(_PILEUP)?%s$
127  """ % suffix , re.VERBOSE)
128 
129 
130 
131  #print logfile_name
132  result = universal_candle_step_regs[suffix].search(filename)
133  if result:
134  #print result.groups()
135  #print "result: %s" % str(result.groups())
136  candle = result.groups()[0]
137  step = result.groups()[1]
138  is_pileup = result.groups()[2]
139  if is_pileup:
140  is_pileup = "PILEUP"
141  else:
142  is_pileup = ""
143 
144  """ if we had the candle without underscore inside (like TTBAR but not E_1000)
145  on the end of result and underscore which needs to be removed """
146 
147  if (candle[-1] == '_'):
148  candle = candle[0:-1]
149 
150  """ try to fetch the conditions and real pileup type if the SimulationCandles.txt is existing """
151  conditions = ''
152  event_content = ''
153  try:
154  conf = read_ConfigurationFromSimulationCandles(path = path, step = step, is_pileup= is_pileup)
155  if conf:
156  is_pileup = conf["pileup_type"]
157  conditions = conf["conditions"]
158  event_content = conf["event_content"]
159  except OSError, e:
160  pass
161 
162  return (candle, step, is_pileup, conditions, event_content)
163  else:
164  return (None, None, None, None, None)
165 
def read_ConfigurationFromSimulationCandles
def FileNamesHelper.getJobID_fromRootFileName (   logfile_name)
Returns the candle and STEP out of filename:

* the candle might include one optional underscore:
>>> getJobID_fromRootFileName("PI-_1000_GEN,SIM.root")
('PI-_1000', 'GEN,SIM', '', '')

* otherwise after candle we have two underscores:
>>> getJobID_fromRootFileName("MINBIAS__GEN,FASTSIM.root")
('MINBIAS', 'GEN,FASTSIM', '', '')

* and lastly we have the PILEUP possibility:
>>> getJobID_fromRootFileName("TTBAR__DIGI_PILEUP.root")
('TTBAR', 'DIGI', 'PILEUP', '')

Definition at line 166 of file FileNamesHelper.py.

References getJobID_fromFileName().

Referenced by getRootFileSize().

167 def getJobID_fromRootFileName(logfile_name):
168  """
169  Returns the candle and STEP out of filename:
170 
171  * the candle might include one optional underscore:
172  >>> getJobID_fromRootFileName("PI-_1000_GEN,SIM.root")
173  ('PI-_1000', 'GEN,SIM', '', '')
174 
175  * otherwise after candle we have two underscores:
176  >>> getJobID_fromRootFileName("MINBIAS__GEN,FASTSIM.root")
177  ('MINBIAS', 'GEN,FASTSIM', '', '')
178 
179  * and lastly we have the PILEUP possibility:
180  >>> getJobID_fromRootFileName("TTBAR__DIGI_PILEUP.root")
181  ('TTBAR', 'DIGI', 'PILEUP', '')
182  """
183  return getJobID_fromFileName(logfile_name, "\\.root")
def FileNamesHelper.getJobID_fromTimeReportLogName (   logfile_name)
Returns the candle and STEP out of filename:

* the candle might include one optional underscore:
>>> getJobID_fromTimeReportLogName("E_1000_GEN,SIM_TimingReport.log")
('E_1000', 'GEN,SIM', '', '')

* otherwise after candle we have two underscores:
>>> getJobID_fromTimeReportLogName("test_data/TTBAR__RAW2DIGI,RECO_TimingReport.log")
('TTBAR', 'RAW2DIGI,RECO', '', '')

* and lastly we have the PILEUP possibility:
>>> getJobID_fromTimeReportLogName("TTBAR__DIGI_PILEUP_TimingReport.log")
('TTBAR', 'DIGI', 'PILEUP', '')

Definition at line 202 of file FileNamesHelper.py.

References getJobID_fromFileName().

Referenced by parserTimingReport.manual_run(), parserTimingReport.perf_profile(), and cmsPerfSuiteHarvest.process_timesize_dir().

203 def getJobID_fromTimeReportLogName(logfile_name):
204  """
205  Returns the candle and STEP out of filename:
206 
207  * the candle might include one optional underscore:
208  >>> getJobID_fromTimeReportLogName("E_1000_GEN,SIM_TimingReport.log")
209  ('E_1000', 'GEN,SIM', '', '')
210 
211  * otherwise after candle we have two underscores:
212  >>> getJobID_fromTimeReportLogName("test_data/TTBAR__RAW2DIGI,RECO_TimingReport.log")
213  ('TTBAR', 'RAW2DIGI,RECO', '', '')
214 
215  * and lastly we have the PILEUP possibility:
216  >>> getJobID_fromTimeReportLogName("TTBAR__DIGI_PILEUP_TimingReport.log")
217  ('TTBAR', 'DIGI', 'PILEUP', '')
218  """
219  return getJobID_fromFileName(logfile_name, "_TimingReport.log")
220 
221 
""" Get the root file size for the candle, step in current dir """
def getJobID_fromTimeReportLogName
def FileNamesHelper.getRootFileSize (   path,
  candle,
  step 
)

Definition at line 222 of file FileNamesHelper.py.

References f_candle_and_step_inJobID, and getJobID_fromRootFileName().

Referenced by cmsPerfSuiteHarvest.process_timesize_dir().

223 def getRootFileSize(path, candle, step):
224  files = os.listdir(path)
225  root_files = [os.path.join(path, f) for f in files
226  if test_root_file.search(f)
227  and os.path.isfile(os.path.join(path, f)) ]
228 
229  """ get the size of file if it is the root file for current candle and step """
230  try:
231  size = [os.stat(f).st_size for f in root_files
233  except Exception, e:
234  print e
235  return 0
236  return size
list f_candle_and_step_inJobID
def FileNamesHelper.read_ConfigurationFromSimulationCandles (   path,
  step,
  is_pileup 
)

Definition at line 45 of file FileNamesHelper.py.

References parsingRulesHelper.rulesParser(), split, and strip().

Referenced by getJobID_fromFileName(), and cmsPerfSuiteHarvest.process_timesize_dir().

45 
46 def read_ConfigurationFromSimulationCandles(path, step, is_pileup):
47  # Here we parse SimulationCandles_<version: e.g. CMSSW_3_2_0>.txt which contains
48  # release:TODO, release_base [path] - we can put it to release [but it's of different granularity]
49  # how to reproduce stuff: TODO
50 
51  try:
52  """ get the acual file """
53  SimulationCandles_file = [os.path.join(path, f) for f in os.listdir(path)
54  if os.path.isfile(os.path.join(path, f)) and f.startswith("SimulationCandles_")][0]
55  except IndexError:
56  return None
57 
58  """ read and parse it; format: #Version : CMSSW_3_2_0 """
59  f = open(SimulationCandles_file, 'r')
60 
61  lines = [s.strip() for s in f.readlines()]
62  f.close()
63 
64 
65 
66  """ we call a shared helper to parse the file """
67 
68  for line in lines:
69  #print line
70  #print simCandlesRules[2][1].match(line) and simCandlesRules[2][1].match(line).groups() or ""
71 
72  info, missing_fields = parsingRulesHelper.rulesParser(simCandlesRules, [line], compileRules = False)
73  #Massaging the info dictionary conditions entry to allow for new cmsDriver.py --conditions option:
74  if 'auto:' in info['conditions']:
75  from Configuration.PyReleaseValidation.autoCond import autoCond
76  info['conditions'] = autoCond[ info['conditions'].split(':')[1] ].split("::")[0]
77  else:
78  if 'FrontierConditions_GlobalTag' in info['conditions']:
79  info['conditions']=info['conditions'].split(",")[1]
80  #print (info, missing_fields)
81  #if we successfully parsed the line of simulation candles:
82  if not missing_fields:
83  #we have to match only step and
84  if info["step"].strip() == step.strip() and ((not is_pileup and not info["pileup_type"]) or (is_pileup and info["pileup_type"])):
85  # if it's pile up or not:
86  #print "Info for <<%s, %s>>: %s" % (str(step), str(is_pileup), str(info))
87  return info
88 
89 
90 
91 
void strip(std::string &input, const std::string &blanks=" \n\t")
Definition: stringTools.cc:16
def read_ConfigurationFromSimulationCandles
double split
Definition: MVATrainer.cc:139
def FileNamesHelper.read_SimulationCandles (   path)
get the acual file 

Definition at line 237 of file FileNamesHelper.py.

Referenced by cmsPerfSuiteHarvest.process_timesize_dir().

238 def read_SimulationCandles(path):
239  # Here we parse SimulationCandles_<version: e.g. CMSSW_3_2_0>.txt which contains
240  # release:TODO, release_base [path] - we can put it to release [but it's of different granularity]
241  # how to reproduce stuff: TODO
242 
243  """ get the acual file """
244  SimulationCandles_file = [os.path.join(path, f) for f in os.listdir(path)
245  if os.path.isfile(os.path.join(path, f)) and f.startswith("SimulationCandles_")][0]
246 
247  """ read and parse it; format: #Version : CMSSW_3_2_0 """
248  f = open(SimulationCandles_file, 'r')
249  lines = f.readlines()
250  f.close()
251 
252  release_version =[[a.strip() for a in line.split(":")] for line in lines if line.startswith("#Version")][0][1]
253  return release_version
254 

Variable Documentation

list FileNamesHelper.f_candle_and_step_inJobID = lambdacandle,step,x:x[0]

Definition at line 6 of file FileNamesHelper.py.

Referenced by parserEdmSize.getEdmReport(), and getRootFileSize().

string FileNamesHelper.path = "/home/vidma/Desktop/CERN_code/cmssw/data/CMSSW_3_2_0_--usersteps=GEN-SIM,DIGI_lxbuild106.cern.ch_relval/relval/CMSSW_3_2_0/workGENSIMDIGI/TTbar_PU_TimeSize"

Definition at line 258 of file FileNamesHelper.py.

tuple FileNamesHelper.simCandlesRules
Initial value:
1 = (
2 
3  #e.g.: --conditions FrontierConditions_GlobalTag,MC_31X_V4::All --eventcontent RECOSIM
4  (("cms_driver_options", ), r"""^cmsDriver.py(.+)$"""),
5  #Changing the following to allow for new cmsDriver.py --conditions option (that can optionally drop the FrontierConditions_GlobalTag,)
6  (("", "conditions", ""), r"""^cmsDriver.py(.*)--conditions ([^\s]+)(.*)$""", "req"),
7  (("", "pileup_type", ""), r"""^cmsDriver.py(.*)--pileup=([^\s]+)(.*)$"""),
8  (("", "step", ""), r"""^cmsDriver.py(.*)--step=([^\s]+)(.*)$""", "req"),
9  #not shure if event content is required
10  (("", "event_content", ""), r"""^cmsDriver.py(.*)--eventcontent ([^\s]+)(.*)$""", "req"),
11  (("", "num_events", ""), r"""^cmsDriver.py(.*)-n ([^\s]+)(.*)$""", "req"),
12 
13  #TODO: after changeing the splitter to "taskset -c ..." this is no longer included into the part of correct job
14  #(("input_user_root_file", ), r"""^For these tests will use user input file (.+)$"""),
15 )

Definition at line 28 of file FileNamesHelper.py.

tuple FileNamesHelper.test_root_file = re.compile(".root$", re.IGNORECASE)

Definition at line 19 of file FileNamesHelper.py.

dictionary FileNamesHelper.universal_candle_step_regs = {}

Definition at line 18 of file FileNamesHelper.py.