CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
utils Namespace Reference

Classes

class  BinToBin
 
class  BinToBin1percent
 
class  Chi2
 
class  KS
 
class  StatisticalTest
 
class  unpickler
 

Functions

def ask_ok
 
def code_generator
 
def functor
 
def get_relval_cmssw_version
 
def get_relval_id
 
def get_relval_max_version
 
def get_relval_version
 -------------—— Make files pairs: RelVal utils ---------------—— More...
 
def get_relvaldata_cmssw_version
 
def get_relvaldata_id
 -----------—— Make files pairs: RelValData utils --------------—— More...
 
def get_relvaldata_max_version
 
def get_relvaldata_version
 
def getNbins
 
def is_empty
 
def is_relvaldata
 ----------------------— Make files pairs -----------------------— More...
 
def is_sparse
 
def literal2root
 
def logger
 
def make_files_pairs
 
def parse_word
 
def profile2histo
 
def setTDRStyle
 
def test_env
 
def tree
 
def user_info
 
def wget
 

Variables

int _log_level = 10
 
tuple TAG = re.compile(r'[a-zA-Z0-9]')
 
 theargv = sys.argv
 

Function Documentation

def utils.ask_ok (   prompt,
  retries = 4,
  complaint = 'yes or no' 
)

Definition at line 423 of file utils.py.

423 
424 def ask_ok(prompt, retries=4, complaint='yes or no'):
425  while True:
426  ok = raw_input(prompt)
427  if ok in ('y', 'ye', 'yes'):
428  return True
429  if ok in ('n', 'no'):
430  return False
431  retries = retries - 1
432  if retries < 0:
433  raise IOError('refusenik user')
434  print complaint
435 
436 #-------------------------------------------------------------------------------
utils.ask_ok
def ask_ok
Definition: utils.py:423
def utils.code_generator (   kwds) 
Code generator function, parse user arguments, load and
return appropriate template generator module.

Definition at line 106 of file utils.py.

Referenced by cms.generate(), and main.generator().

106 
107 def code_generator(kwds):
108  """
109  Code generator function, parse user arguments, load and
110  return appropriate template generator module.
111  """
112  debug = kwds.get('debug', None)
113  if debug:
114  print "Configuration:"
115  pprint.pprint(kwds)
116  try:
117  klass = kwds.get('tmpl')
118  mname = 'FWCore.Skeletons.%s' % klass.lower()
119  module = __import__(mname, fromlist=[klass])
120  except ImportError as err:
121  klass = 'AbstractPkg'
122  module = __import__('FWCore.Skeletons.pkg', fromlist=[klass])
123  if debug:
124  print "%s, will use %s" % (str(err), klass)
125  obj = getattr(module, klass)(kwds)
126  return obj
utils.code_generator
def code_generator
Definition: utils.py:106
def utils.functor (   code,
  kwds,
  debug = 0 
) 
Auto-generate and execute function with given code and configuration
For details of compile/exec/eval see
http://lucumr.pocoo.org/2011/2/1/exec-in-python/

Definition at line 54 of file utils.py.

References join().

Referenced by pkg.AbstractPkg.write().

54 
55 def functor(code, kwds, debug=0):
56  """
57  Auto-generate and execute function with given code and configuration
58  For details of compile/exec/eval see
59  http://lucumr.pocoo.org/2011/2/1/exec-in-python/
60  """
61  args = []
62  for key, val in kwds.items():
63  if isinstance(val, basestring):
64  arg = '%s="%s"' % (key, val)
65  elif isinstance(val, list):
66  arg = '%s=%s' % (key, val)
67  else:
68  msg = 'Unsupported data type "%s" <%s>' % (val, type(val))
69  raise Exception(msg)
70  args.append(arg)
71  func = '\nimport sys'
72  func += '\nimport StringIO'
73  func += "\ndef func(%s):\n" % ','.join(args)
74  func += code
75  func += """
76 def capture():
77  "Capture snippet printous"
78  old_stdout = sys.stdout
79  sys.stdout = StringIO.StringIO()
80  func()
81  out = sys.stdout.getvalue()
82  sys.stdout = old_stdout
83  return out\n
84 capture()\n"""
85  if debug:
86  print "\n### generated code\n"
87  print func
88  # compile python code as exec statement
89  obj = compile(func, '<string>', 'exec')
90  # define execution namespace
91  namespace = {}
92  # execute compiled python code in given namespace
93  exec obj in namespace
94  # located generated function object, run it and return its results
95  return namespace['capture']()
utils.functor
def functor
Definition: utils.py:54
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def utils.get_relval_cmssw_version (   file)

Definition at line 536 of file utils.py.

536 
537 def get_relval_cmssw_version(file):
538  cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
539  gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-([\w\d]*)_V\d*\w?(_[\w\d]*)?-v', file)
540  if cmssw_release and gr_r_version:
541  return (cmssw_release[0], gr_r_version[0])
utils.get_relval_cmssw_version
def get_relval_cmssw_version
Definition: utils.py:536
def utils.get_relval_id (   file) 
Returns unique relval ID (dataset name) for a given file.

Definition at line 542 of file utils.py.

542 
543 def get_relval_id(file):
544  """Returns unique relval ID (dataset name) for a given file."""
545  dataset_name = re.findall('R\d{9}__([\w\d]*)__CMSSW_', file)
546  return dataset_name[0]
utils.get_relval_id
def get_relval_id
Definition: utils.py:542
def utils.get_relval_max_version (   files) 
Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 524 of file utils.py.

References get_relval_version().

524 
525 def get_relval_max_version(files):
526  """Returns file with maximum version at a) beggining of the file,
527  e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
528  max_file = files[0]
529  max_v = get_relval_version(files[0])
530  for file in files:
531  file_v = get_relval_version(file)
532  if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
533  max_file = file
534  max_v = file_v
535  return max_file
utils.get_relval_version
def get_relval_version
-------------—— Make files pairs: RelVal utils ---------------——
Definition: utils.py:517
utils.get_relval_max_version
def get_relval_max_version
Definition: utils.py:524
def utils.get_relval_version (   file)

-------------—— Make files pairs: RelVal utils ---------------—— 

Returns tuple (CMSSW version, run version) for specified file.

Definition at line 517 of file utils.py.

Referenced by get_relval_max_version().

517 
518 def get_relval_version(file):
519  """Returns tuple (CMSSW version, run version) for specified file."""
520  cmssw_version = re.findall('DQM_V(\d*)_', file)
521  run_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-[\w\d]*_V\d*\w?(?:_[\w\d]*)?-v(\d*)__', file)
522  if cmssw_version and run_version:
523  return (int(cmssw_version[0]), int(run_version[0]))
utils.get_relval_version
def get_relval_version
-------------—— Make files pairs: RelVal utils ---------------——
Definition: utils.py:517
def utils.get_relvaldata_cmssw_version (   file) 
Returns tuple (CMSSW release, GR_R version) for specified RelValData file.

Definition at line 486 of file utils.py.

486 
487 def get_relvaldata_cmssw_version(file):
488  """Returns tuple (CMSSW release, GR_R version) for specified RelValData file."""
489  cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
490  gr_r_version = re.findall('-(GR_R_\d*_V\d*\w?)(?:_RelVal)?_', file)
491  if not gr_r_version:
492  gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-(\w*)_RelVal_', file)
493  if cmssw_release and gr_r_version:
494  return (cmssw_release[0], gr_r_version[0])
utils.get_relvaldata_cmssw_version
def get_relvaldata_cmssw_version
Definition: utils.py:486
def utils.get_relvaldata_id (   file)

-----------—— Make files pairs: RelValData utils --------------—— 

Returns unique relvaldata ID for a given file.

Definition at line 476 of file utils.py.

476 
477 def get_relvaldata_id(file):
478  """Returns unique relvaldata ID for a given file."""
479  run_id = re.search('R\d{9}', file)
480  run = re.search('_RelVal_([\w\d]*)-v\d__', file)
481  if not run:
482  run = re.search('GR_R_\d*_V\d*C?_([\w\d]*)-v\d__', file)
483  if run_id and run:
484  return (run_id.group(), run.group(1))
485  return None
utils.get_relvaldata_id
def get_relvaldata_id
-----------—— Make files pairs: RelValData utils --------------——
Definition: utils.py:476
def utils.get_relvaldata_max_version (   files) 
Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 504 of file utils.py.

References get_relvaldata_version().

504 
505 def get_relvaldata_max_version(files):
506  """Returns file with maximum version at a) beggining of the file,
507  e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
508  max_file = files[0]
509  max_v = get_relvaldata_version(files[0])
510  for file in files:
511  file_v = get_relvaldata_version(file)
512  if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
513  max_file = file
514  max_v = file_v
515  return max_file
utils.get_relvaldata_version
def get_relvaldata_version
Definition: utils.py:495
utils.get_relvaldata_max_version
def get_relvaldata_max_version
Definition: utils.py:504
def utils.get_relvaldata_version (   file) 
Returns tuple (CMSSW version, run version) for specified file.

Definition at line 495 of file utils.py.

Referenced by get_relvaldata_max_version().

495 
496 def get_relvaldata_version(file):
497  """Returns tuple (CMSSW version, run version) for specified file."""
498  cmssw_version = re.findall('DQM_V(\d*)_', file)
499  run_version = re.findall('_RelVal_[\w\d]*-v(\d)__', file)
500  if not run_version:
501  run_version = re.findall('GR_R_\d*_V\d*C?_[\w\d]*-v(\d)__', file)
502  if cmssw_version and run_version:
503  return (int(cmssw_version[0]), int(run_version[0]))
utils.get_relvaldata_version
def get_relvaldata_version
Definition: utils.py:495
def utils.getNbins (   h)

Definition at line 92 of file utils.py.

Referenced by utils.Chi2.absval(), utils.BinToBin.do_test(), utils.BinToBin1percent.do_test(), and utils.StatisticalTest.get_rank().

92 
93 def getNbins(h):
94  biny=h.GetNbinsY()
95  if biny>1:biny+=1
96  binz=h.GetNbinsZ()
97  if binz>1:binz+=1
98  return (h.GetNbinsX()+1)*(biny)*(binz)
99 
100 #-------------------------------------------------------------------------------
101 
utils.getNbins
def getNbins
Definition: utils.py:92
def utils.is_empty (   h)

Definition at line 171 of file utils.py.

Referenced by DTSectColl.addTSTheta(), DEcompare< T >.get_ncand(), utils.StatisticalTest.get_rank(), and DEcompare< T >.SortCollections().

171 
172 def is_empty(h):
173  for i in xrange(1,getNbins(h)):
174  if h.GetBinContent(i)!=0: return False
175  return True
176  #return h.GetSumOfWeights()==0
177 
178 #-------------------------------------------------------------------------------
utils.is_empty
def is_empty
Definition: utils.py:171
utils.getNbins
def getNbins
Definition: utils.py:92
def utils.is_relvaldata (   files)

----------------------— Make files pairs -----------------------—

Definition at line 548 of file utils.py.

References any.

Referenced by make_files_pairs().

548 
549 def is_relvaldata(files):
550  is_relvaldata_re = re.compile('_RelVal_')
551  return any([is_relvaldata_re.search(filename) for filename in files])
utils.is_relvaldata
def is_relvaldata
----------------------— Make files pairs -----------------------—
Definition: utils.py:548
def utils.is_sparse (   h)

Definition at line 179 of file utils.py.

179 
180 def is_sparse(h):
181  filled_bins=0.
182  nbins=h.GetNbinsX()
183  for ibin in xrange(nbins):
184  if h.GetBinContent(ibin)>0:
185  filled_bins+=1
186  #print "%s %s --> %s" %(filled_bins,nbins,filled_bins/nbins)
187  if filled_bins/nbins < .5:
188  return True
189  else:
190  return False
191 
192 #-------------------------------------------------------------------------------
utils.is_sparse
def is_sparse
Definition: utils.py:179
def utils.literal2root (   literal,
  rootType 
)

Definition at line 69 of file utils.py.

69 
70 def literal2root (literal,rootType):
71  bitsarray = array.array('B')
72  bitsarray.fromstring(literal.decode('hex'))
73 
74  tbuffer=0
75  try:
76  tbuffer = TBufferFile(TBufferFile.kRead, len(bitsarray), bitsarray, False,0)
77  except:
78  print "could not transform to object array:"
79  print [ i for i in bitsarray ]
80 
81  # replace a couple of shortcuts with the real root class name
82  if rootType == 'TPROF':
83  rootType = 'TProfile'
84  if rootType == 'TPROF2D':
85  rootType = 'TProfile2D'
86 
87  root_class=eval(rootType+'.Class()')
88 
89  return tbuffer.ReadObject(root_class)
90 
91 #-------------------------------------------------------------------------------
utils.literal2root
def literal2root
Definition: utils.py:69
def utils.logger (   msg_level,
  message 
)

Definition at line 47 of file utils.py.

Referenced by utils.StatisticalTest.get_rank(), and utils.StatisticalTest.get_status().

47 
48 def logger(msg_level,message):
49  if msg_level>=_log_level:
50  print "[%s] %s" %(asctime(),message)
51 
#-------------------------------------------------------------------------------
utils.logger
def logger
Definition: utils.py:47
def utils.make_files_pairs (   files,
  verbose = True 
)

Definition at line 552 of file utils.py.

References python.multivaluedict.append(), python.multivaluedict.dict, cmsRelvalreport.exit, is_relvaldata(), and max().

Referenced by ValidationMatrix.get_filenames_from_pool().

552 
553 def make_files_pairs(files, verbose=True):
554  ## Select functions to use
555  if is_relvaldata(files):
556  is_relval_data = True
557  get_cmssw_version = get_relvaldata_cmssw_version
558  get_id = get_relvaldata_id
559  get_max_version = get_relvaldata_max_version
560  # print 'Pairing Data RelVal files.'
561  else:
562  is_relval_data = False
563  get_cmssw_version = get_relval_cmssw_version
564  get_id = get_relval_id
565  get_max_version = get_relval_max_version
566  # print 'Pairing Monte Carlo RelVal files.'
567 
568  ## Divide files into groups
569  versions_files = dict()
570  for file in files:
571  version = get_cmssw_version(file)
572  if versions_files.has_key(version):
573  versions_files[version].append(file)
574  else:
575  versions_files[version] = [file]
576 
577  ## Print the division into groups
578  if verbose:
579  print '\nFound versions:'
580  for version in versions_files:
581  print '%s: %d files' % (str(version), len(versions_files[version]))
582 
583  if len(versions_files.keys()) <= 1:
584  print '\nFound too little versions, there is nothing to pair. Exiting...\n'
585  exit()
586 
587  ## Select two biggest groups.
588  versions = versions_files.keys()
589  sizes = [len(value) for value in versions_files.values()]
590  v1 = versions[sizes.index(max(sizes))]
591  versions.remove(v1)
592  sizes.remove(max(sizes))
593  v2 = versions[sizes.index(max(sizes))]
594 
595  ## Print two biggest groups.
596  if verbose:
597  print '\nPairing %s (%d files) and %s (%d files)' % (str(v1),
598  len(versions_files[v1]), str(v2), len(versions_files[v2]))
599 
600  ## Pairing two versions
601  print '\nGot pairs:'
602  pairs = []
603  for unique_id in set([get_id(file) for file in versions_files[v1]]):
604  if is_relval_data:
605  dataset_re = re.compile(unique_id[0]+'_')
606  run_re = re.compile(unique_id[1])
607  c1_files = [file for file in versions_files[v1] if dataset_re.search(file) and run_re.search(file)]
608  c2_files = [file for file in versions_files[v2] if dataset_re.search(file) and run_re.search(file)]
609  else:
610  dataset_re = re.compile(unique_id+'_')
611  c1_files = [file for file in versions_files[v1] if dataset_re.search(file)]
612  c2_files = [file for file in versions_files[v2] if dataset_re.search(file)]
613 
614  if len(c1_files) > 0 and len(c2_files) > 0:
615  first_file = get_max_version(c1_files)
616  second_file = get_max_version(c2_files)
617  print '%s\n%s\n' % (first_file, second_file)
618  pairs.extend((first_file, second_file))
619  if verbose:
620  print "Paired and got %d files.\n" % len(pairs)
621  return pairs
max
const T & max(const T &a, const T &b)
Definition: MaterialBudgetTrackerHistos.cc:4
utils.is_relvaldata
def is_relvaldata
----------------------— Make files pairs -----------------------—
Definition: utils.py:548
utils.make_files_pairs
def make_files_pairs
Definition: utils.py:552
def utils.parse_word (   word)

Definition at line 21 of file utils.py.

Referenced by pkg.AbstractPkg.tmpl_tags().

21 
22 def parse_word(word):
23  "Parse word which contas double underscore tag"
24  output = set()
25  words = word.split()
26  for idx in xrange(0, len(words)):
27  pat = words[idx]
28  if pat and len(pat) > 4 and pat[:2] == '__': # we found enclosure
29  tag = pat[2:pat.rfind('__')]
30  if tag.find('__') != -1: # another pattern
31  for item in tag.split('__'):
32  if TAG.match(item):
33  output.add('__%s__' % item)
34  else:
35  output.add('__%s__' % tag)
36  return output
utils.parse_word
def parse_word
Definition: utils.py:21
def utils.profile2histo (   profile)

Definition at line 220 of file utils.py.

220 
221 def profile2histo(profile):
222  if not profile.InheritsFrom("TH1"):
223  return profile
224 
225  bin_low_edges=[]
226  n_bins=profile.GetNbinsX()
227 
228  for ibin in xrange(1,n_bins+2):
229  bin_low_edges.append(profile.GetBinLowEdge(ibin))
230  bin_low_edges=array.array('f',bin_low_edges)
231  histo=TH1F(profile.GetName(),profile.GetTitle(),n_bins,bin_low_edges)
232  for ibin in xrange(0,n_bins+1):
233  histo.SetBinContent(ibin,profile.GetBinContent(ibin))
234  histo.SetBinError(ibin,profile.GetBinError(ibin))
235 
236  return histo
237 #-------------------------------------------------------------------------------
utils.profile2histo
def profile2histo
Definition: utils.py:220
def utils.setTDRStyle ( )

Definition at line 52 of file utils.py.

References plotscripts.tdrStyle.

Referenced by FlavourHistograms< T >.plot(), and TrackIPTagPlotter.psPlot().

52 
53 def setTDRStyle():
54  this_dir=dirname(this_module_name)
55  this_dir_one_up=this_dir[:this_dir.rfind("/")+1]
56  #this_dir_two_up=this_dir_one_up[:this_dir_one_up.rfind("/")+1]
57  style_file=''
58  if os.environ.has_key("RELMON_SA"):
59  style_file=this_dir_one_up+"data/tdrstyle_mod.C"
60  else:
61  style_file="%s/src/Utilities/RelMon/data/tdrstyle_mod.C"%(os.environ["CMSSW_BASE"])
62  try:
63  gROOT.ProcessLine(".L %s" %style_file)
64  gROOT.ProcessLine("setTDRStyle()")
65  except:
66  "Print could not set the TDR style. File %s not found?" %style_file
67 
68 
#-------------------------------------------------------------------------------
utils.setTDRStyle
def setTDRStyle
Definition: utils.py:52
def utils.test_env (   tdir,
  tmpl 
) 
Test user environment, look-up if user has run cmsenv, otherwise
provide meaningful error message back to the user.

Definition at line 37 of file utils.py.

Referenced by main.generator().

37 
38 def test_env(tdir, tmpl):
39  """
40  Test user environment, look-up if user has run cmsenv, otherwise
41  provide meaningful error message back to the user.
42  """
43  if not tdir or not os.path.isdir(tdir):
44  print "Unable to access template dir: %s" % tdir
45  sys.exit(1)
46  if not os.listdir(tdir):
47  print "No template files found in template dir %s" % tdir
48  sys.exit(0)
49  if not tmpl:
50  msg = "No template type is provided, "
51  msg += "see available templates via --templates option"
52  print msg
53  sys.exit(1)
utils.test_env
def test_env
Definition: utils.py:37
def utils.tree (   idir)

Definition at line 127 of file utils.py.

127 
128 def tree(idir):
129  "Print directory content, similar to tree UNIX command"
130  if idir[-1] == '/':
131  idir = idir[-1]
132  dsep = ''
133  fsep = ''
134  dtot = -1 # we'll not count initial directory
135  ftot = 0
136  for root, dirs, files in os.walk(idir):
137  dirs = root.split('/')
138  ndirs = len(dirs)
139  if ndirs > 1:
140  dsep = '| '*(ndirs-1)
141  print '%s%s/' % (dsep, dirs[-1])
142  dtot += 1
143  for fname in files:
144  fsep = dsep + '|--'
145  print '%s %s' % (fsep, fname)
146  ftot += 1
147  if dtot == -1 or not dtot:
148  dmsg = ''
149  else:
150  dmsg = '%s directories,' % dtot
151  if ftot:
152  fmsg = '%s file' % ftot
153  if ftot > 1:
154  fmsg += 's'
155  else:
156  fmsg = ''
157  if dmsg and fmsg:
158  print "Total: %s %s" % (dmsg, fmsg)
159  else:
160  print "No directories/files in %s" % idir
utils.tree
def tree
Definition: utils.py:127
def utils.user_info (   ainput = None)

Definition at line 96 of file utils.py.

Referenced by cond.userInfo().

96 
97 def user_info(ainput=None):
98  "Return user name and office location, based on UNIX finger"
99  if ainput:
100  return ainput
101  pwdstr = pwd.getpwnam(os.getlogin())
102  author = pwdstr.pw_gecos
103  if author and isinstance(author, basestring):
104  author = author.split(',')[0]
105  return author
utils.user_info
def user_info
Definition: utils.py:96
def utils.wget (   url) 
Fetch the WHOLE file, not in bunches... To be optimised.

Definition at line 451 of file utils.py.

References SiPixelLorentzAngle_cfi.read.

451 
452 def wget(url):
453  """ Fetch the WHOLE file, not in bunches... To be optimised.
454  """
455  opener=build_opener(X509CertOpen())
456  datareq = Request(url)
457  datareq.add_header('authenticated_wget', "The ultimate wgetter")
458  bin_content=None
459  try:
460  filename=basename(url)
461  print "Checking existence of file %s on disk..."%filename
462  if not isfile("./%s"%filename):
463  bin_content=opener.open(datareq).read()
464  else:
465  print "File %s exists, skipping.." %filename
466  except ValueError:
467  print "Error: Unknown url %s" %url
468 
469  if bin_content!=None:
470  ofile = open(filename, 'wb')
471  ofile.write(bin_content)
472  ofile.close()
473 
#-------------------------------------------------------------------------------
utils.wget
def wget
Definition: utils.py:451
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7

Variable Documentation

int utils._log_level = 10

Definition at line 46 of file utils.py.

tuple utils.TAG = re.compile(r'[a-zA-Z0-9]')

Definition at line 19 of file utils.py.

utils.theargv = sys.argv

Definition at line 24 of file utils.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5