test
CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
utils Namespace Reference

Classes

class  BinToBin
 
class  BinToBin1percent
 
class  Chi2
 
class  KS
 
class  StatisticalTest
 
class  unpickler
 

Functions

def ask_ok
 
def code_generator
 
def functor
 
def get_relval_cmssw_version
 
def get_relval_id
 
def get_relval_max_version
 
def get_relval_version
 -------------—— Make files pairs: RelVal utils ---------------—— More...
 
def get_relvaldata_cmssw_version
 
def get_relvaldata_id
 -----------—— Make files pairs: RelValData utils --------------—— More...
 
def get_relvaldata_max_version
 
def get_relvaldata_version
 
def getNbins
 
def is_empty
 
def is_relvaldata
 ----------------------— Make files pairs -----------------------— More...
 
def is_sparse
 
def literal2root
 
def logger
 
def make_files_pairs
 
def parse_word
 
def profile2histo
 
def setTDRStyle
 
def test_env
 
def tree
 
def user_info
 
def wget
 

Variables

int _log_level = 10
 
tuple TAG = re.compile(r'[a-zA-Z0-9]')
 
 theargv = sys.argv
 

Function Documentation

def utils.ask_ok (   prompt,
  retries = 4,
  complaint = 'yes or no' 
)

Definition at line 419 of file utils.py.

420 def ask_ok(prompt, retries=4, complaint='yes or no'):
421  while True:
422  ok = raw_input(prompt)
423  if ok in ('y', 'ye', 'yes'):
424  return True
425  if ok in ('n', 'no'):
426  return False
427  retries = retries - 1
428  if retries < 0:
429  raise IOError('refusenik user')
430  print complaint
431 
432 #-------------------------------------------------------------------------------
def ask_ok
Definition: utils.py:419
def utils.code_generator (   kwds)
Code generator function, parse user arguments, load and
return appropriate template generator module.

Definition at line 106 of file utils.py.

Referenced by cms.generate(), and main.generator().

107 def code_generator(kwds):
108  """
109  Code generator function, parse user arguments, load and
110  return appropriate template generator module.
111  """
112  debug = kwds.get('debug', None)
113  if debug:
114  print "Configuration:"
115  pprint.pprint(kwds)
116  try:
117  klass = kwds.get('tmpl')
118  mname = 'FWCore.Skeletons.%s' % klass.lower()
119  module = __import__(mname, fromlist=[klass])
120  except ImportError as err:
121  klass = 'AbstractPkg'
122  module = __import__('FWCore.Skeletons.pkg', fromlist=[klass])
123  if debug:
124  print "%s, will use %s" % (str(err), klass)
125  obj = getattr(module, klass)(kwds)
126  return obj
def code_generator
Definition: utils.py:106
def utils.functor (   code,
  kwds,
  debug = 0 
)
Auto-generate and execute function with given code and configuration
For details of compile/exec/eval see
http://lucumr.pocoo.org/2011/2/1/exec-in-python/

Definition at line 54 of file utils.py.

References join().

Referenced by pkg.AbstractPkg.write().

54 
55 def functor(code, kwds, debug=0):
56  """
57  Auto-generate and execute function with given code and configuration
58  For details of compile/exec/eval see
59  http://lucumr.pocoo.org/2011/2/1/exec-in-python/
60  """
61  args = []
62  for key, val in kwds.items():
63  if isinstance(val, basestring):
64  arg = '%s="%s"' % (key, val)
65  elif isinstance(val, list):
66  arg = '%s=%s' % (key, val)
67  else:
68  msg = 'Unsupported data type "%s" <%s>' % (val, type(val))
69  raise Exception(msg)
70  args.append(arg)
71  func = '\nimport sys'
72  func += '\nimport StringIO'
73  func += "\ndef func(%s):\n" % ','.join(args)
74  func += code
75  func += """
76 def capture():
77  "Capture snippet printous"
78  old_stdout = sys.stdout
79  sys.stdout = StringIO.StringIO()
80  func()
81  out = sys.stdout.getvalue()
82  sys.stdout = old_stdout
83  return out\n
84 capture()\n"""
85  if debug:
86  print "\n### generated code\n"
87  print func
88  # compile python code as exec statement
89  obj = compile(func, '<string>', 'exec')
90  # define execution namespace
91  namespace = {}
92  # execute compiled python code in given namespace
93  exec obj in namespace
94  # located generated function object, run it and return its results
95  return namespace['capture']()
def functor
Definition: utils.py:54
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def utils.get_relval_cmssw_version (   file)

Definition at line 532 of file utils.py.

533 def get_relval_cmssw_version(file):
534  cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
535  gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-([\w\d]*)_V\d*\w?(_[\w\d]*)?-v', file)
536  if cmssw_release and gr_r_version:
537  if "PU" in gr_r_version[0][0] and not "FastSim" in file:
538  __gt = re.sub('^[^_]*_', "", gr_r_version[0][0])
539  __process_string = gr_r_version[0][1]
540  return (__gt, __process_string)
541  elif "PU" in gr_r_version[0][0] and "FastSim" in file: #a check for FastSimPU samples
542  return (cmssw_release[0], "PU_") #with possibly different GT's
543  return (cmssw_release[0], gr_r_version[0])
def get_relval_cmssw_version
Definition: utils.py:532
def utils.get_relval_id (   file)
Returns unique relval ID (dataset name) for a given file.

Definition at line 544 of file utils.py.

References watchdog.group.

545 def get_relval_id(file):
546  """Returns unique relval ID (dataset name) for a given file."""
547  dataset_name = re.findall('R\d{9}__([\w\D]*)__CMSSW_', file)
548  __process_string = re.search('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-([\w\d]*)_V\d*\w?(_[\w\d]*)?-v', file)
549  _ps = ""
550  if __process_string:
551  if "PU" in __process_string.group(1) and not "FastSim" in file:
552  _ps = re.search('^[^_]*_', __process_string.group(1)).group()
553  elif "PU" in __process_string.group(1) and "FastSim" in file:
554  return dataset_name[0]+"_", _ps ##some testing is needed
555  return dataset_name[0], _ps
tuple group
Definition: watchdog.py:82
def get_relval_id
Definition: utils.py:544
def utils.get_relval_max_version (   files)
Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 520 of file utils.py.

References get_relval_version().

521 def get_relval_max_version(files):
522  """Returns file with maximum version at a) beggining of the file,
523  e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
524  max_file = files[0]
525  max_v = get_relval_version(files[0])
526  for file in files:
527  file_v = get_relval_version(file)
528  if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
529  max_file = file
530  max_v = file_v
531  return max_file
def get_relval_version
-------------—— Make files pairs: RelVal utils ---------------——
Definition: utils.py:513
def get_relval_max_version
Definition: utils.py:520
def utils.get_relval_version (   file)

-------------—— Make files pairs: RelVal utils ---------------——

Returns tuple (CMSSW version, run version) for specified file.

Definition at line 513 of file utils.py.

Referenced by get_relval_max_version().

514 def get_relval_version(file):
515  """Returns tuple (CMSSW version, run version) for specified file."""
516  cmssw_version = re.findall('DQM_V(\d*)_', file)
517  run_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-[\w\d]*_V\d*\w?(?:_[\w\d]*)?-v(\d*)__', file)
518  if cmssw_version and run_version:
519  return (int(cmssw_version[0]), int(run_version[0]))
def get_relval_version
-------------—— Make files pairs: RelVal utils ---------------——
Definition: utils.py:513
def utils.get_relvaldata_cmssw_version (   file)
Returns tuple (CMSSW release, GR_R version) for specified RelValData file.

Definition at line 482 of file utils.py.

484  """Returns tuple (CMSSW release, GR_R version) for specified RelValData file."""
485  cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
486  gr_r_version = re.findall('-(GR_R_\d*_V\d*\w?)(?:_RelVal)?_', file)
487  if not gr_r_version:
488  gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-(\w*)_RelVal_', file)
489  if cmssw_release and gr_r_version:
490  return (cmssw_release[0], gr_r_version[0])
def get_relvaldata_cmssw_version
Definition: utils.py:482
def utils.get_relvaldata_id (   file)

-----------—— Make files pairs: RelValData utils --------------——

Returns unique relvaldata ID for a given file.

Definition at line 472 of file utils.py.

473 def get_relvaldata_id(file):
474  """Returns unique relvaldata ID for a given file."""
475  run_id = re.search('R\d{9}', file)
476  run = re.search('_RelVal_([\w\d]*)-v\d__', file)
477  if not run:
478  run = re.search('GR_R_\d*_V\d*C?_([\w\d]*)-v\d__', file)
479  if run_id and run:
480  return (run_id.group(), run.group(1))
481  return None
def get_relvaldata_id
-----------—— Make files pairs: RelValData utils --------------——
Definition: utils.py:472
def utils.get_relvaldata_max_version (   files)
Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 500 of file utils.py.

References get_relvaldata_version().

501 def get_relvaldata_max_version(files):
502  """Returns file with maximum version at a) beggining of the file,
503  e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
504  max_file = files[0]
505  max_v = get_relvaldata_version(files[0])
506  for file in files:
507  file_v = get_relvaldata_version(file)
508  if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
509  max_file = file
510  max_v = file_v
511  return max_file
def get_relvaldata_version
Definition: utils.py:491
def get_relvaldata_max_version
Definition: utils.py:500
def utils.get_relvaldata_version (   file)
Returns tuple (CMSSW version, run version) for specified file.

Definition at line 491 of file utils.py.

Referenced by get_relvaldata_max_version().

492 def get_relvaldata_version(file):
493  """Returns tuple (CMSSW version, run version) for specified file."""
494  cmssw_version = re.findall('DQM_V(\d*)_', file)
495  run_version = re.findall('_RelVal_[\w\d]*-v(\d)__', file)
496  if not run_version:
497  run_version = re.findall('GR_R_\d*_V\d*C?_[\w\d]*-v(\d)__', file)
498  if cmssw_version and run_version:
499  return (int(cmssw_version[0]), int(run_version[0]))
def get_relvaldata_version
Definition: utils.py:491
def utils.getNbins (   h)

Definition at line 89 of file utils.py.

Referenced by utils.Chi2.absval(), utils.BinToBin.do_test(), utils.BinToBin1percent.do_test(), and utils.StatisticalTest.get_rank().

89 
90 def getNbins(h):
91  biny=h.GetNbinsY()
92  if biny>1:biny+=1
93  binz=h.GetNbinsZ()
94  if binz>1:binz+=1
95  return (h.GetNbinsX()+1)*(biny)*(binz)
96 
97 #-------------------------------------------------------------------------------
98 
def getNbins
Definition: utils.py:89
def utils.is_empty (   h)

Definition at line 167 of file utils.py.

Referenced by DTSectColl.addTSTheta(), DEcompare< T >.get_ncand(), utils.StatisticalTest.get_rank(), and DEcompare< T >.SortCollections().

168 def is_empty(h):
169  for i in xrange(1,getNbins(h)):
170  if h.GetBinContent(i)!=0: return False
171  return True
172  #return h.GetSumOfWeights()==0
173 
174 #-------------------------------------------------------------------------------
def is_empty
Definition: utils.py:167
def getNbins
Definition: utils.py:89
def utils.is_relvaldata (   files)

----------------------— Make files pairs -----------------------—

Definition at line 557 of file utils.py.

References any().

Referenced by make_files_pairs().

558 def is_relvaldata(files):
559  is_relvaldata_re = re.compile('_RelVal_')
560  return any([is_relvaldata_re.search(filename) for filename in files])
bool any(const std::vector< T > &v, const T &what)
Definition: ECalSD.cc:34
def is_relvaldata
----------------------— Make files pairs -----------------------—
Definition: utils.py:557
def utils.is_sparse (   h)

Definition at line 175 of file utils.py.

176 def is_sparse(h):
177  filled_bins=0.
178  nbins=h.GetNbinsX()
179  for ibin in xrange(nbins):
180  if h.GetBinContent(ibin)>0:
181  filled_bins+=1
182  #print "%s %s --> %s" %(filled_bins,nbins,filled_bins/nbins)
183  if filled_bins/nbins < .5:
184  return True
185  else:
186  return False
187 
188 #-------------------------------------------------------------------------------
def is_sparse
Definition: utils.py:175
def utils.literal2root (   literal,
  rootType 
)

Definition at line 66 of file utils.py.

66 
67 def literal2root (literal,rootType):
68  bitsarray = array.array('B')
69  bitsarray.fromstring(literal.decode('hex'))
70 
71  tbuffer=0
72  try:
73  tbuffer = TBufferFile(TBufferFile.kRead, len(bitsarray), bitsarray, False,0)
74  except:
75  print "could not transform to object array:"
76  print [ i for i in bitsarray ]
77 
78  # replace a couple of shortcuts with the real root class name
79  if rootType == 'TPROF':
80  rootType = 'TProfile'
81  if rootType == 'TPROF2D':
82  rootType = 'TProfile2D'
83 
84  root_class=eval(rootType+'.Class()')
85 
86  return tbuffer.ReadObject(root_class)
87 
88 #-------------------------------------------------------------------------------
def literal2root
Definition: utils.py:66
def utils.logger (   msg_level,
  message 
)

Definition at line 44 of file utils.py.

44 
45 def logger(msg_level,message):
46  if msg_level>=_log_level:
47  print "[%s] %s" %(asctime(),message)
48 
#-------------------------------------------------------------------------------
def logger
Definition: utils.py:44
def utils.make_files_pairs (   files,
  verbose = True 
)

Definition at line 561 of file utils.py.

References bitset_utilities.append(), cmsPerfStripChart.dict, cmsRelvalreport.exit, is_relvaldata(), and bookConverter.max.

Referenced by ValidationMatrix.get_filenames_from_pool().

562 def make_files_pairs(files, verbose=True):
563  ## Select functions to use
564  if is_relvaldata(files):
565  is_relval_data = True
566  get_cmssw_version = get_relvaldata_cmssw_version
567  get_id = get_relvaldata_id
568  get_max_version = get_relvaldata_max_version
569  # print 'Pairing Data RelVal files.'
570  else:
571  is_relval_data = False
572  get_cmssw_version = get_relval_cmssw_version
573  get_id = get_relval_id
574  get_max_version = get_relval_max_version
575  # print 'Pairing Monte Carlo RelVal files.'
576 
577  ## Divide files into groups
578  versions_files = dict()
579  for file in files:
580  version = get_cmssw_version(file)
581  if version in versions_files:
582  versions_files[version].append(file)
583  else:
584  versions_files[version] = [file]
585 
586  ## Print the division into groups
587  if verbose:
588  print '\nFound versions:'
589  for version in versions_files:
590  print '%s: %d files' % (str(version), len(versions_files[version]))
591 
592  if len(versions_files.keys()) <= 1:
593  print '\nFound too little versions, there is nothing to pair. Exiting...\n'
594  exit()
595 
596  ## Select two biggest groups.
597  versions = versions_files.keys()
598  sizes = [len(value) for value in versions_files.values()]
599  v1 = versions[sizes.index(max(sizes))]
600  versions.remove(v1)
601  sizes.remove(max(sizes))
602  v2 = versions[sizes.index(max(sizes))]
603 
604  ## Print two biggest groups.
605  if verbose:
606  print '\nPairing %s (%d files) and %s (%d files)' % (str(v1),
607  len(versions_files[v1]), str(v2), len(versions_files[v2]))
608 
609  ## Pairing two versions
610  print '\nGot pairs:'
611  pairs = []
612  for unique_id in set([get_id(file) for file in versions_files[v1]]):
613  if is_relval_data:
614  dataset_re = re.compile(unique_id[0]+'_')
615  run_re = re.compile(unique_id[1])
616  c1_files = [file for file in versions_files[v1] if dataset_re.search(file) and run_re.search(file)]
617  c2_files = [file for file in versions_files[v2] if dataset_re.search(file) and run_re.search(file)]
618  else:
619  dataset_re = re.compile(unique_id[0]+'_')
620  ps_re = re.compile(unique_id[1])
621  ##compile a PU re and search also for same PU
622  c1_files = [file for file in versions_files[v1] if dataset_re.search(file) and ps_re.search(file)]
623  c2_files = [file for file in versions_files[v2] if dataset_re.search(file) and ps_re.search(file)]
624 
625  if len(c1_files) > 0 and len(c2_files) > 0:
626  first_file = get_max_version(c1_files)
627  second_file = get_max_version(c2_files)
628  print '%s\n%s\n' % (first_file, second_file)
629  pairs.extend((first_file, second_file))
630  if verbose:
631  print "Paired and got %d files.\n" % len(pairs)
632  return pairs
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
def is_relvaldata
----------------------— Make files pairs -----------------------—
Definition: utils.py:557
def make_files_pairs
Definition: utils.py:561
def utils.parse_word (   word)

Definition at line 21 of file utils.py.

Referenced by pkg.AbstractPkg.tmpl_tags().

21 
22 def parse_word(word):
23  "Parse word which contas double underscore tag"
24  output = set()
25  words = word.split()
26  for idx in xrange(0, len(words)):
27  pat = words[idx]
28  if pat and len(pat) > 4 and pat[:2] == '__': # we found enclosure
29  tag = pat[2:pat.rfind('__')]
30  if tag.find('__') != -1: # another pattern
31  for item in tag.split('__'):
32  if TAG.match(item):
33  output.add('__%s__' % item)
34  else:
35  output.add('__%s__' % tag)
36  return output
def parse_word
Definition: utils.py:21
def utils.profile2histo (   profile)

Definition at line 216 of file utils.py.

217 def profile2histo(profile):
218  if not profile.InheritsFrom("TH1"):
219  return profile
220 
221  bin_low_edges=[]
222  n_bins=profile.GetNbinsX()
223 
224  for ibin in xrange(1,n_bins+2):
225  bin_low_edges.append(profile.GetBinLowEdge(ibin))
226  bin_low_edges=array.array('f',bin_low_edges)
227  histo=TH1F(profile.GetName(),profile.GetTitle(),n_bins,bin_low_edges)
228  for ibin in xrange(0,n_bins+1):
229  histo.SetBinContent(ibin,profile.GetBinContent(ibin))
230  histo.SetBinError(ibin,profile.GetBinError(ibin))
231 
232  return histo
233 #-------------------------------------------------------------------------------
def profile2histo
Definition: utils.py:216
def utils.setTDRStyle ( )

Definition at line 49 of file utils.py.

References HILowLumiHLTOfflineSource_cfi.dirname, and plotscripts.tdrStyle.

Referenced by FlavourHistograms< T >.plot(), FlavourHistograms2D< T, G >.plot(), and IPTagPlotter< Container, Base >.psPlot().

49 
50 def setTDRStyle():
51  this_dir=dirname(this_module_name)
52  this_dir_one_up=this_dir[:this_dir.rfind("/")+1]
53  #this_dir_two_up=this_dir_one_up[:this_dir_one_up.rfind("/")+1]
54  style_file=''
55  if "RELMON_SA" in os.environ:
56  style_file=this_dir_one_up+"data/tdrstyle_mod.C"
57  else:
58  style_file="%s/src/Utilities/RelMon/data/tdrstyle_mod.C"%(os.environ["CMSSW_BASE"])
59  try:
60  gROOT.ProcessLine(".L %s" %style_file)
61  gROOT.ProcessLine("setTDRStyle()")
62  except:
63  "Print could not set the TDR style. File %s not found?" %style_file
64 
65 
#-------------------------------------------------------------------------------
def setTDRStyle
Definition: utils.py:49
def utils.test_env (   tdir,
  tmpl 
)
Test user environment, look-up if user has run cmsenv, otherwise
provide meaningful error message back to the user.

Definition at line 37 of file utils.py.

Referenced by main.generator().

37 
38 def test_env(tdir, tmpl):
39  """
40  Test user environment, look-up if user has run cmsenv, otherwise
41  provide meaningful error message back to the user.
42  """
43  if not tdir or not os.path.isdir(tdir):
44  print "Unable to access template dir: %s" % tdir
45  sys.exit(1)
46  if not os.listdir(tdir):
47  print "No template files found in template dir %s" % tdir
48  sys.exit(0)
49  if not tmpl:
50  msg = "No template type is provided, "
51  msg += "see available templates via --templates option"
52  print msg
53  sys.exit(1)
def test_env
Definition: utils.py:37
def utils.tree (   idir)

Definition at line 127 of file utils.py.

128 def tree(idir):
129  "Print directory content, similar to tree UNIX command"
130  if idir[-1] == '/':
131  idir = idir[-1]
132  dsep = ''
133  fsep = ''
134  dtot = -1 # we'll not count initial directory
135  ftot = 0
136  for root, dirs, files in os.walk(idir):
137  dirs = root.split('/')
138  ndirs = len(dirs)
139  if ndirs > 1:
140  dsep = '| '*(ndirs-1)
141  print '%s%s/' % (dsep, dirs[-1])
142  dtot += 1
143  for fname in files:
144  fsep = dsep + '|--'
145  print '%s %s' % (fsep, fname)
146  ftot += 1
147  if dtot == -1 or not dtot:
148  dmsg = ''
149  else:
150  dmsg = '%s directories,' % dtot
151  if ftot:
152  fmsg = '%s file' % ftot
153  if ftot > 1:
154  fmsg += 's'
155  else:
156  fmsg = ''
157  if dmsg and fmsg:
158  print "Total: %s %s" % (dmsg, fmsg)
159  else:
160  print "No directories/files in %s" % idir
def tree
Definition: utils.py:127
def utils.user_info (   ainput = None)

Definition at line 96 of file utils.py.

96 
97 def user_info(ainput=None):
98  "Return user name and office location, based on UNIX finger"
99  if ainput:
100  return ainput
101  pwdstr = pwd.getpwnam(os.getlogin())
102  author = pwdstr.pw_gecos
103  if author and isinstance(author, basestring):
104  author = author.split(',')[0]
105  return author
def user_info
Definition: utils.py:96
def utils.wget (   url)
Fetch the WHOLE file, not in bunches... To be optimised.

Definition at line 447 of file utils.py.

References SiPixelLorentzAngle_cfi.read.

448 def wget(url):
449  """ Fetch the WHOLE file, not in bunches... To be optimised.
450  """
451  opener=build_opener(X509CertOpen())
452  datareq = Request(url)
453  datareq.add_header('authenticated_wget', "The ultimate wgetter")
454  bin_content=None
455  try:
456  filename=basename(url)
457  print "Checking existence of file %s on disk..."%filename
458  if not isfile("./%s"%filename):
459  bin_content=opener.open(datareq).read()
460  else:
461  print "File %s exists, skipping.." %filename
462  except ValueError:
463  print "Error: Unknown url %s" %url
464 
465  if bin_content!=None:
466  ofile = open(filename, 'wb')
467  ofile.write(bin_content)
468  ofile.close()
469 
#-------------------------------------------------------------------------------
def wget
Definition: utils.py:447

Variable Documentation

int utils._log_level = 10

Definition at line 43 of file utils.py.

tuple utils.TAG = re.compile(r'[a-zA-Z0-9]')

Definition at line 19 of file utils.py.

utils.theargv = sys.argv

Definition at line 21 of file utils.py.