CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
utils Namespace Reference

Classes

class  BinToBin
 
class  BinToBin1percent
 
class  Chi2
 
class  KS
 
class  StatisticalTest
 
class  unpickler
 

Functions

def ask_ok
 
def code_generator
 
def functor
 
def get_relval_cmssw_version
 
def get_relval_id
 
def get_relval_max_version
 
def get_relval_version
 -------------—— Make files pairs: RelVal utils ---------------—— More...
 
def get_relvaldata_cmssw_version
 
def get_relvaldata_id
 -----------—— Make files pairs: RelValData utils --------------—— More...
 
def get_relvaldata_max_version
 
def get_relvaldata_version
 
def getNbins
 
def is_empty
 
def is_relvaldata
 ----------------------— Make files pairs -----------------------— More...
 
def is_sparse
 
def literal2root
 
def logger
 
def make_files_pairs
 
def parse_word
 
def profile2histo
 
def setTDRStyle
 
def test_env
 
def tree
 
def user_info
 
def wget
 

Variables

int _log_level = 10
 
tuple TAG = re.compile(r'[a-zA-Z0-9]')
 
 theargv = sys.argv
 

Function Documentation

def utils.ask_ok (   prompt,
  retries = 4,
  complaint = 'yes or no' 
)

Definition at line 420 of file utils.py.

420 
421 def ask_ok(prompt, retries=4, complaint='yes or no'):
422  while True:
423  ok = raw_input(prompt)
424  if ok in ('y', 'ye', 'yes'):
425  return True
426  if ok in ('n', 'no'):
427  return False
428  retries = retries - 1
429  if retries < 0:
430  raise IOError('refusenik user')
431  print complaint
432 
433 #-------------------------------------------------------------------------------
utils.ask_ok
def ask_ok
Definition: utils.py:420
def utils.code_generator (   kwds) 
Code generator function, parse user arguments, load and
return appropriate template generator module.

Definition at line 106 of file utils.py.

Referenced by cms.generate(), and main.generator().

106 
107 def code_generator(kwds):
108  """
109  Code generator function, parse user arguments, load and
110  return appropriate template generator module.
111  """
112  debug = kwds.get('debug', None)
113  if debug:
114  print "Configuration:"
115  pprint.pprint(kwds)
116  try:
117  klass = kwds.get('tmpl')
118  mname = 'FWCore.Skeletons.%s' % klass.lower()
119  module = __import__(mname, fromlist=[klass])
120  except ImportError as err:
121  klass = 'AbstractPkg'
122  module = __import__('FWCore.Skeletons.pkg', fromlist=[klass])
123  if debug:
124  print "%s, will use %s" % (str(err), klass)
125  obj = getattr(module, klass)(kwds)
126  return obj
utils.code_generator
def code_generator
Definition: utils.py:106
def utils.functor (   code,
  kwds,
  debug = 0 
) 
Auto-generate and execute function with given code and configuration
For details of compile/exec/eval see
http://lucumr.pocoo.org/2011/2/1/exec-in-python/

Definition at line 54 of file utils.py.

References join().

Referenced by pkg.AbstractPkg.write().

54 
55 def functor(code, kwds, debug=0):
56  """
57  Auto-generate and execute function with given code and configuration
58  For details of compile/exec/eval see
59  http://lucumr.pocoo.org/2011/2/1/exec-in-python/
60  """
61  args = []
62  for key, val in kwds.items():
63  if isinstance(val, basestring):
64  arg = '%s="%s"' % (key, val)
65  elif isinstance(val, list):
66  arg = '%s=%s' % (key, val)
67  else:
68  msg = 'Unsupported data type "%s" <%s>' % (val, type(val))
69  raise Exception(msg)
70  args.append(arg)
71  func = '\nimport sys'
72  func += '\nimport StringIO'
73  func += "\ndef func(%s):\n" % ','.join(args)
74  func += code
75  func += """
76 def capture():
77  "Capture snippet printous"
78  old_stdout = sys.stdout
79  sys.stdout = StringIO.StringIO()
80  func()
81  out = sys.stdout.getvalue()
82  sys.stdout = old_stdout
83  return out\n
84 capture()\n"""
85  if debug:
86  print "\n### generated code\n"
87  print func
88  # compile python code as exec statement
89  obj = compile(func, '<string>', 'exec')
90  # define execution namespace
91  namespace = {}
92  # execute compiled python code in given namespace
93  exec obj in namespace
94  # located generated function object, run it and return its results
95  return namespace['capture']()
utils.functor
def functor
Definition: utils.py:54
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def utils.get_relval_cmssw_version (   file)

Definition at line 533 of file utils.py.

533 
534 def get_relval_cmssw_version(file):
535  cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
536  gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-([\w\d]*)_V\d*\w?(_[\w\d]*)?-v', file)
537  if cmssw_release and gr_r_version:
538  return (cmssw_release[0], gr_r_version[0])
utils.get_relval_cmssw_version
def get_relval_cmssw_version
Definition: utils.py:533
def utils.get_relval_id (   file) 
Returns unique relval ID (dataset name) for a given file.

Definition at line 539 of file utils.py.

539 
540 def get_relval_id(file):
541  """Returns unique relval ID (dataset name) for a given file."""
542  dataset_name = re.findall('R\d{9}__([\w\d]*)__CMSSW_', file)
543  return dataset_name[0]
utils.get_relval_id
def get_relval_id
Definition: utils.py:539
def utils.get_relval_max_version (   files) 
Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 521 of file utils.py.

References get_relval_version().

521 
522 def get_relval_max_version(files):
523  """Returns file with maximum version at a) beggining of the file,
524  e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
525  max_file = files[0]
526  max_v = get_relval_version(files[0])
527  for file in files:
528  file_v = get_relval_version(file)
529  if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
530  max_file = file
531  max_v = file_v
532  return max_file
utils.get_relval_version
def get_relval_version
-------------—— Make files pairs: RelVal utils ---------------——
Definition: utils.py:514
utils.get_relval_max_version
def get_relval_max_version
Definition: utils.py:521
def utils.get_relval_version (   file)

-------------—— Make files pairs: RelVal utils ---------------—— 

Returns tuple (CMSSW version, run version) for specified file.

Definition at line 514 of file utils.py.

Referenced by get_relval_max_version().

514 
515 def get_relval_version(file):
516  """Returns tuple (CMSSW version, run version) for specified file."""
517  cmssw_version = re.findall('DQM_V(\d*)_', file)
518  run_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-[\w\d]*_V\d*\w?(?:_[\w\d]*)?-v(\d*)__', file)
519  if cmssw_version and run_version:
520  return (int(cmssw_version[0]), int(run_version[0]))
utils.get_relval_version
def get_relval_version
-------------—— Make files pairs: RelVal utils ---------------——
Definition: utils.py:514
def utils.get_relvaldata_cmssw_version (   file) 
Returns tuple (CMSSW release, GR_R version) for specified RelValData file.

Definition at line 483 of file utils.py.

483 
484 def get_relvaldata_cmssw_version(file):
485  """Returns tuple (CMSSW release, GR_R version) for specified RelValData file."""
486  cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
487  gr_r_version = re.findall('-(GR_R_\d*_V\d*\w?)(?:_RelVal)?_', file)
488  if not gr_r_version:
489  gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-(\w*)_RelVal_', file)
490  if cmssw_release and gr_r_version:
491  return (cmssw_release[0], gr_r_version[0])
utils.get_relvaldata_cmssw_version
def get_relvaldata_cmssw_version
Definition: utils.py:483
def utils.get_relvaldata_id (   file)

-----------—— Make files pairs: RelValData utils --------------—— 

Returns unique relvaldata ID for a given file.

Definition at line 473 of file utils.py.

473 
474 def get_relvaldata_id(file):
475  """Returns unique relvaldata ID for a given file."""
476  run_id = re.search('R\d{9}', file)
477  run = re.search('_RelVal_([\w\d]*)-v\d__', file)
478  if not run:
479  run = re.search('GR_R_\d*_V\d*C?_([\w\d]*)-v\d__', file)
480  if run_id and run:
481  return (run_id.group(), run.group(1))
482  return None
utils.get_relvaldata_id
def get_relvaldata_id
-----------—— Make files pairs: RelValData utils --------------——
Definition: utils.py:473
def utils.get_relvaldata_max_version (   files) 
Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 501 of file utils.py.

References get_relvaldata_version().

501 
502 def get_relvaldata_max_version(files):
503  """Returns file with maximum version at a) beggining of the file,
504  e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
505  max_file = files[0]
506  max_v = get_relvaldata_version(files[0])
507  for file in files:
508  file_v = get_relvaldata_version(file)
509  if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
510  max_file = file
511  max_v = file_v
512  return max_file
utils.get_relvaldata_version
def get_relvaldata_version
Definition: utils.py:492
utils.get_relvaldata_max_version
def get_relvaldata_max_version
Definition: utils.py:501
def utils.get_relvaldata_version (   file) 
Returns tuple (CMSSW version, run version) for specified file.

Definition at line 492 of file utils.py.

Referenced by get_relvaldata_max_version().

492 
493 def get_relvaldata_version(file):
494  """Returns tuple (CMSSW version, run version) for specified file."""
495  cmssw_version = re.findall('DQM_V(\d*)_', file)
496  run_version = re.findall('_RelVal_[\w\d]*-v(\d)__', file)
497  if not run_version:
498  run_version = re.findall('GR_R_\d*_V\d*C?_[\w\d]*-v(\d)__', file)
499  if cmssw_version and run_version:
500  return (int(cmssw_version[0]), int(run_version[0]))
utils.get_relvaldata_version
def get_relvaldata_version
Definition: utils.py:492
def utils.getNbins (   h)

Definition at line 89 of file utils.py.

Referenced by utils.Chi2.absval(), utils.BinToBin.do_test(), utils.BinToBin1percent.do_test(), and utils.StatisticalTest.get_rank().

89 
90 def getNbins(h):
91  biny=h.GetNbinsY()
92  if biny>1:biny+=1
93  binz=h.GetNbinsZ()
94  if binz>1:binz+=1
95  return (h.GetNbinsX()+1)*(biny)*(binz)
96 
97 #-------------------------------------------------------------------------------
98 
utils.getNbins
def getNbins
Definition: utils.py:89
def utils.is_empty (   h)

Definition at line 168 of file utils.py.

Referenced by DTSectColl.addTSTheta(), DEcompare< T >.get_ncand(), utils.StatisticalTest.get_rank(), and DEcompare< T >.SortCollections().

168 
169 def is_empty(h):
170  for i in xrange(1,getNbins(h)):
171  if h.GetBinContent(i)!=0: return False
172  return True
173  #return h.GetSumOfWeights()==0
174 
175 #-------------------------------------------------------------------------------
utils.is_empty
def is_empty
Definition: utils.py:168
utils.getNbins
def getNbins
Definition: utils.py:89
def utils.is_relvaldata (   files)

----------------------— Make files pairs -----------------------—

Definition at line 545 of file utils.py.

References any().

Referenced by make_files_pairs().

545 
546 def is_relvaldata(files):
547  is_relvaldata_re = re.compile('_RelVal_')
548  return any([is_relvaldata_re.search(filename) for filename in files])
any
bool any(const std::vector< T > &v, const T &what)
Definition: ECalSD.cc:34
utils.is_relvaldata
def is_relvaldata
----------------------— Make files pairs -----------------------—
Definition: utils.py:545
def utils.is_sparse (   h)

Definition at line 176 of file utils.py.

176 
177 def is_sparse(h):
178  filled_bins=0.
179  nbins=h.GetNbinsX()
180  for ibin in xrange(nbins):
181  if h.GetBinContent(ibin)>0:
182  filled_bins+=1
183  #print "%s %s --> %s" %(filled_bins,nbins,filled_bins/nbins)
184  if filled_bins/nbins < .5:
185  return True
186  else:
187  return False
188 
189 #-------------------------------------------------------------------------------
utils.is_sparse
def is_sparse
Definition: utils.py:176
def utils.literal2root (   literal,
  rootType 
)

Definition at line 66 of file utils.py.

66 
67 def literal2root (literal,rootType):
68  bitsarray = array.array('B')
69  bitsarray.fromstring(literal.decode('hex'))
70 
71  tbuffer=0
72  try:
73  tbuffer = TBufferFile(TBufferFile.kRead, len(bitsarray), bitsarray, False,0)
74  except:
75  print "could not transform to object array:"
76  print [ i for i in bitsarray ]
77 
78  # replace a couple of shortcuts with the real root class name
79  if rootType == 'TPROF':
80  rootType = 'TProfile'
81  if rootType == 'TPROF2D':
82  rootType = 'TProfile2D'
83 
84  root_class=eval(rootType+'.Class()')
85 
86  return tbuffer.ReadObject(root_class)
87 
88 #-------------------------------------------------------------------------------
utils.literal2root
def literal2root
Definition: utils.py:66
def utils.logger (   msg_level,
  message 
)

Definition at line 44 of file utils.py.

Referenced by utils.StatisticalTest.get_rank(), and utils.StatisticalTest.get_status().

44 
45 def logger(msg_level,message):
46  if msg_level>=_log_level:
47  print "[%s] %s" %(asctime(),message)
48 
#-------------------------------------------------------------------------------
utils.logger
def logger
Definition: utils.py:44
def utils.make_files_pairs (   files,
  verbose = True 
)

Definition at line 549 of file utils.py.

References python.multivaluedict.append(), python.multivaluedict.dict, cmsRelvalreport.exit, is_relvaldata(), and max().

Referenced by ValidationMatrix.get_filenames_from_pool().

549 
550 def make_files_pairs(files, verbose=True):
551  ## Select functions to use
552  if is_relvaldata(files):
553  is_relval_data = True
554  get_cmssw_version = get_relvaldata_cmssw_version
555  get_id = get_relvaldata_id
556  get_max_version = get_relvaldata_max_version
557  # print 'Pairing Data RelVal files.'
558  else:
559  is_relval_data = False
560  get_cmssw_version = get_relval_cmssw_version
561  get_id = get_relval_id
562  get_max_version = get_relval_max_version
563  # print 'Pairing Monte Carlo RelVal files.'
564 
565  ## Divide files into groups
566  versions_files = dict()
567  for file in files:
568  version = get_cmssw_version(file)
569  if versions_files.has_key(version):
570  versions_files[version].append(file)
571  else:
572  versions_files[version] = [file]
573 
574  ## Print the division into groups
575  if verbose:
576  print '\nFound versions:'
577  for version in versions_files:
578  print '%s: %d files' % (str(version), len(versions_files[version]))
579 
580  if len(versions_files.keys()) <= 1:
581  print '\nFound too little versions, there is nothing to pair. Exiting...\n'
582  exit()
583 
584  ## Select two biggest groups.
585  versions = versions_files.keys()
586  sizes = [len(value) for value in versions_files.values()]
587  v1 = versions[sizes.index(max(sizes))]
588  versions.remove(v1)
589  sizes.remove(max(sizes))
590  v2 = versions[sizes.index(max(sizes))]
591 
592  ## Print two biggest groups.
593  if verbose:
594  print '\nPairing %s (%d files) and %s (%d files)' % (str(v1),
595  len(versions_files[v1]), str(v2), len(versions_files[v2]))
596 
597  ## Pairing two versions
598  print '\nGot pairs:'
599  pairs = []
600  for unique_id in set([get_id(file) for file in versions_files[v1]]):
601  if is_relval_data:
602  dataset_re = re.compile(unique_id[0]+'_')
603  run_re = re.compile(unique_id[1])
604  c1_files = [file for file in versions_files[v1] if dataset_re.search(file) and run_re.search(file)]
605  c2_files = [file for file in versions_files[v2] if dataset_re.search(file) and run_re.search(file)]
606  else:
607  dataset_re = re.compile(unique_id+'_')
608  c1_files = [file for file in versions_files[v1] if dataset_re.search(file)]
609  c2_files = [file for file in versions_files[v2] if dataset_re.search(file)]
610 
611  if len(c1_files) > 0 and len(c2_files) > 0:
612  first_file = get_max_version(c1_files)
613  second_file = get_max_version(c2_files)
614  print '%s\n%s\n' % (first_file, second_file)
615  pairs.extend((first_file, second_file))
616  if verbose:
617  print "Paired and got %d files.\n" % len(pairs)
618  return pairs
max
const T & max(const T &a, const T &b)
Definition: MaterialBudgetTrackerHistos.cc:4
utils.is_relvaldata
def is_relvaldata
----------------------— Make files pairs -----------------------—
Definition: utils.py:545
utils.make_files_pairs
def make_files_pairs
Definition: utils.py:549
def utils.parse_word (   word)

Definition at line 21 of file utils.py.

Referenced by pkg.AbstractPkg.tmpl_tags().

21 
22 def parse_word(word):
23  "Parse word which contas double underscore tag"
24  output = set()
25  words = word.split()
26  for idx in xrange(0, len(words)):
27  pat = words[idx]
28  if pat and len(pat) > 4 and pat[:2] == '__': # we found enclosure
29  tag = pat[2:pat.rfind('__')]
30  if tag.find('__') != -1: # another pattern
31  for item in tag.split('__'):
32  if TAG.match(item):
33  output.add('__%s__' % item)
34  else:
35  output.add('__%s__' % tag)
36  return output
utils.parse_word
def parse_word
Definition: utils.py:21
def utils.profile2histo (   profile)

Definition at line 217 of file utils.py.

217 
218 def profile2histo(profile):
219  if not profile.InheritsFrom("TH1"):
220  return profile
221 
222  bin_low_edges=[]
223  n_bins=profile.GetNbinsX()
224 
225  for ibin in xrange(1,n_bins+2):
226  bin_low_edges.append(profile.GetBinLowEdge(ibin))
227  bin_low_edges=array.array('f',bin_low_edges)
228  histo=TH1F(profile.GetName(),profile.GetTitle(),n_bins,bin_low_edges)
229  for ibin in xrange(0,n_bins+1):
230  histo.SetBinContent(ibin,profile.GetBinContent(ibin))
231  histo.SetBinError(ibin,profile.GetBinError(ibin))
232 
233  return histo
234 #-------------------------------------------------------------------------------
utils.profile2histo
def profile2histo
Definition: utils.py:217
def utils.setTDRStyle ( )

Definition at line 49 of file utils.py.

References plotscripts.tdrStyle.

Referenced by FlavourHistograms< T >.plot(), and TrackIPTagPlotter.psPlot().

49 
50 def setTDRStyle():
51  this_dir=dirname(this_module_name)
52  this_dir_one_up=this_dir[:this_dir.rfind("/")+1]
53  #this_dir_two_up=this_dir_one_up[:this_dir_one_up.rfind("/")+1]
54  style_file=''
55  if os.environ.has_key("RELMON_SA"):
56  style_file=this_dir_one_up+"data/tdrstyle_mod.C"
57  else:
58  style_file="%s/src/Utilities/RelMon/data/tdrstyle_mod.C"%(os.environ["CMSSW_BASE"])
59  try:
60  gROOT.ProcessLine(".L %s" %style_file)
61  gROOT.ProcessLine("setTDRStyle()")
62  except:
63  "Print could not set the TDR style. File %s not found?" %style_file
64 
65 
#-------------------------------------------------------------------------------
utils.setTDRStyle
def setTDRStyle
Definition: utils.py:49
def utils.test_env (   tdir,
  tmpl 
) 
Test user environment, look-up if user has run cmsenv, otherwise
provide meaningful error message back to the user.

Definition at line 37 of file utils.py.

Referenced by main.generator().

37 
38 def test_env(tdir, tmpl):
39  """
40  Test user environment, look-up if user has run cmsenv, otherwise
41  provide meaningful error message back to the user.
42  """
43  if not tdir or not os.path.isdir(tdir):
44  print "Unable to access template dir: %s" % tdir
45  sys.exit(1)
46  if not os.listdir(tdir):
47  print "No template files found in template dir %s" % tdir
48  sys.exit(0)
49  if not tmpl:
50  msg = "No template type is provided, "
51  msg += "see available templates via --templates option"
52  print msg
53  sys.exit(1)
utils.test_env
def test_env
Definition: utils.py:37
def utils.tree (   idir)

Definition at line 127 of file utils.py.

127 
128 def tree(idir):
129  "Print directory content, similar to tree UNIX command"
130  if idir[-1] == '/':
131  idir = idir[-1]
132  dsep = ''
133  fsep = ''
134  dtot = -1 # we'll not count initial directory
135  ftot = 0
136  for root, dirs, files in os.walk(idir):
137  dirs = root.split('/')
138  ndirs = len(dirs)
139  if ndirs > 1:
140  dsep = '| '*(ndirs-1)
141  print '%s%s/' % (dsep, dirs[-1])
142  dtot += 1
143  for fname in files:
144  fsep = dsep + '|--'
145  print '%s %s' % (fsep, fname)
146  ftot += 1
147  if dtot == -1 or not dtot:
148  dmsg = ''
149  else:
150  dmsg = '%s directories,' % dtot
151  if ftot:
152  fmsg = '%s file' % ftot
153  if ftot > 1:
154  fmsg += 's'
155  else:
156  fmsg = ''
157  if dmsg and fmsg:
158  print "Total: %s %s" % (dmsg, fmsg)
159  else:
160  print "No directories/files in %s" % idir
utils.tree
def tree
Definition: utils.py:127
def utils.user_info (   ainput = None)

Definition at line 96 of file utils.py.

Referenced by cond.userInfo().

96 
97 def user_info(ainput=None):
98  "Return user name and office location, based on UNIX finger"
99  if ainput:
100  return ainput
101  pwdstr = pwd.getpwnam(os.getlogin())
102  author = pwdstr.pw_gecos
103  if author and isinstance(author, basestring):
104  author = author.split(',')[0]
105  return author
utils.user_info
def user_info
Definition: utils.py:96
def utils.wget (   url) 
Fetch the WHOLE file, not in bunches... To be optimised.

Definition at line 448 of file utils.py.

References SiPixelLorentzAngle_cfi.read.

448 
449 def wget(url):
450  """ Fetch the WHOLE file, not in bunches... To be optimised.
451  """
452  opener=build_opener(X509CertOpen())
453  datareq = Request(url)
454  datareq.add_header('authenticated_wget', "The ultimate wgetter")
455  bin_content=None
456  try:
457  filename=basename(url)
458  print "Checking existence of file %s on disk..."%filename
459  if not isfile("./%s"%filename):
460  bin_content=opener.open(datareq).read()
461  else:
462  print "File %s exists, skipping.." %filename
463  except ValueError:
464  print "Error: Unknown url %s" %url
465 
466  if bin_content!=None:
467  ofile = open(filename, 'wb')
468  ofile.write(bin_content)
469  ofile.close()
470 
#-------------------------------------------------------------------------------
utils.wget
def wget
Definition: utils.py:448
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7

Variable Documentation

int utils._log_level = 10

Definition at line 43 of file utils.py.

tuple utils.TAG = re.compile(r'[a-zA-Z0-9]')

Definition at line 19 of file utils.py.

utils.theargv = sys.argv

Definition at line 21 of file utils.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5