utils Namespace Reference

Classes
class	BinToBin
class	BinToBin1percent
class	Chi2
class	KS
class	StatisticalTest
class	unpickler
Functions
def	ask_ok
def	code_generator
def	functor
def	get_relval_cmssw_version
def	get_relval_id
def	get_relval_max_version
def	get_relval_version
	------------------- Make files pairs: RelVal utils ---------------------
def	get_relvaldata_cmssw_version
def	get_relvaldata_id
	----------------- Make files pairs: RelValData utils --------------------
def	get_relvaldata_max_version
def	get_relvaldata_version
def	getNbins
def	is_empty
def	is_relvaldata
	------------------------- Make files pairs --------------------------
def	is_sparse
def	literal2root
def	logger
def	make_files_pairs
def	parse_word
def	profile2histo
def	setTDRStyle
def	test_env
def	tree
def	user_info
def	wget
Variables
int	_log_level = 10
tuple	TAG = re.compile(r'[a-zA-Z0-9]')
	theargv = sys.argv

---

Function Documentation

def utils::ask_ok	(		prompt,
			retries = 4,
			complaint = 'yes or no'
	)

Definition at line 423 of file utils.py.

                                                    :
    while True:
        ok = raw_input(prompt)
        if ok in ('y', 'ye', 'yes'):
            return True
        if ok in ('n', 'no'):
            return False
        retries = retries - 1
        if retries < 0:
            raise IOError('refusenik user')
        print complaint

#-------------------------------------------------------------------------------

def utils::code_generator

(

kwds

)

Code generator function, parse user arguments, load and
return appropriate template generator module.

Definition at line 106 of file utils.py.

                        :
    """
    Code generator function, parse user arguments, load and
    return appropriate template generator module.
    """
    debug = kwds.get('debug', None)
    if  debug:
        print "Configuration:"
        pprint.pprint(kwds)
    try:
        klass  = kwds.get('tmpl')
        mname  = 'FWCore.Skeletons.%s' % klass.lower()
        module = __import__(mname, fromlist=[klass])
    except ImportError as err:
        klass  = 'AbstractPkg'
        module = __import__('FWCore.Skeletons.pkg', fromlist=[klass])
        if  debug:
            print "%s, will use %s" % (str(err), klass)
    obj = getattr(module, klass)(kwds)
    return obj

def utils::functor	(		code,
			kwds,
			debug = 0
	)

Auto-generate and execute function with given code and configuration
For details of compile/exec/eval see
http://lucumr.pocoo.org/2011/2/1/exec-in-python/

Definition at line 54 of file utils.py.

                                :
    """
    Auto-generate and execute function with given code and configuration
    For details of compile/exec/eval see
    http://lucumr.pocoo.org/2011/2/1/exec-in-python/
    """
    args  = []
    for key, val in kwds.items():
        if  isinstance(val, basestring):
            arg = '%s="%s"' % (key, val)
        elif isinstance(val, list):
            arg = '%s=%s' % (key, val)
        else:
            msg = 'Unsupported data type "%s" <%s>' % (val, type(val)) 
            raise Exception(msg)
        args.append(arg)
    func  = '\nimport sys'
    func += '\nimport StringIO'
    func += "\ndef func(%s):\n" % ','.join(args)
    func += code
    func += """
def capture():
    "Capture snippet printous"
    old_stdout = sys.stdout
    sys.stdout = StringIO.StringIO()
    func()
    out = sys.stdout.getvalue()
    sys.stdout = old_stdout
    return out\n
capture()\n"""
    if  debug:
        print "\n### generated code\n"
        print func
    # compile python code as exec statement
    obj   = compile(func, '<string>', 'exec')
    # define execution namespace
    namespace = {}
    # execute compiled python code in given namespace
    exec obj in namespace
    # located generated function object, run it and return its results
    return namespace['capture']()

def utils::get_relval_cmssw_version

(

file

)

Definition at line 536 of file utils.py.

                                  :
    cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
    gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-([\w\d]*)_V\d*\w?(_[\w\d]*)?-v', file)
    if cmssw_release and gr_r_version:
        return (cmssw_release[0], gr_r_version[0])

def utils::get_relval_id

(

file

)

Returns unique relval ID (dataset name) for a given file.

Definition at line 542 of file utils.py.

                       :
    """Returns unique relval ID (dataset name) for a given file."""
    dataset_name = re.findall('R\d{9}__([\w\d]*)__CMSSW_', file)
    return dataset_name[0]

def utils::get_relval_max_version

(

files

)

Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 524 of file utils.py.

                                 :
    """Returns file with maximum version at a) beggining of the file,
    e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
    max_file = files[0]
    max_v = get_relval_version(files[0])
    for file in files:
        file_v = get_relval_version(file)
        if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
            max_file = file
            max_v = file_v
    return max_file

def utils::get_relval_version

(

file

)

------------------- Make files pairs: RelVal utils ---------------------

Returns tuple (CMSSW version, run version) for specified file.

Definition at line 517 of file utils.py.

                            :
    """Returns tuple (CMSSW version, run version) for specified file."""
    cmssw_version = re.findall('DQM_V(\d*)_', file)
    run_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-[\w\d]*_V\d*\w?(?:_[\w\d]*)?-v(\d*)__', file)
    if cmssw_version and run_version:
        return (int(cmssw_version[0]), int(run_version[0]))

def utils::get_relvaldata_cmssw_version

(

file

)

Returns tuple (CMSSW release, GR_R version) for specified RelValData file.

Definition at line 486 of file utils.py.

                                      :
    """Returns tuple (CMSSW release, GR_R version) for specified RelValData file."""
    cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
    gr_r_version = re.findall('-(GR_R_\d*_V\d*\w?)(?:_RelVal)?_', file)
    if not gr_r_version:
        gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-(\w*)_RelVal_', file)
    if cmssw_release and gr_r_version:
        return (cmssw_release[0], gr_r_version[0])

def utils::get_relvaldata_id

(

file

)

----------------- Make files pairs: RelValData utils --------------------

Returns unique relvaldata ID for a given file.

Definition at line 476 of file utils.py.

                           :
    """Returns unique relvaldata ID for a given file."""
    run_id = re.search('R\d{9}', file)
    run = re.search('_RelVal_([\w\d]*)-v\d__', file)
    if not run:
        run = re.search('GR_R_\d*_V\d*C?_([\w\d]*)-v\d__', file)
    if run_id and run:
        return (run_id.group(), run.group(1))
    return None

def utils::get_relvaldata_max_version

(

files

)

Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 504 of file utils.py.

                                     :
    """Returns file with maximum version at a) beggining of the file,
    e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
    max_file = files[0]
    max_v = get_relvaldata_version(files[0])
    for file in files:
        file_v = get_relvaldata_version(file)
        if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
            max_file = file
            max_v = file_v
    return max_file

def utils::get_relvaldata_version

(

file

)

Returns tuple (CMSSW version, run version) for specified file.

Definition at line 495 of file utils.py.

                                :
    """Returns tuple (CMSSW version, run version) for specified file."""
    cmssw_version = re.findall('DQM_V(\d*)_', file)
    run_version = re.findall('_RelVal_[\w\d]*-v(\d)__', file)
    if not run_version:
        run_version = re.findall('GR_R_\d*_V\d*C?_[\w\d]*-v(\d)__', file)
    if cmssw_version and run_version:
        return (int(cmssw_version[0]), int(run_version[0]))

def utils::getNbins

(

h

)

Definition at line 92 of file utils.py.

               :
  biny=h.GetNbinsY()
  if biny>1:biny+=1
  binz=h.GetNbinsZ()
  if binz>1:binz+=1
  return (h.GetNbinsX()+1)*(biny)*(binz)

#-------------------------------------------------------------------------------

def utils::is_empty

(

h

)

Definition at line 171 of file utils.py.

Referenced by DTSectColl::addTSTheta(), DEcompare< T >::get_ncand(), and DEcompare< T >::SortCollections().

               :
  for i in xrange(1,getNbins(h)):
    if h.GetBinContent(i)!=0: return False
  return True
  #return h.GetSumOfWeights()==0

#-------------------------------------------------------------------------------

def utils::is_relvaldata

(

files

)

------------------------- Make files pairs --------------------------

Definition at line 548 of file utils.py.

                        :
    is_relvaldata_re = re.compile('_RelVal_')
    return any([is_relvaldata_re.search(filename) for filename in files])

def utils::is_sparse

(

h

)

Definition at line 179 of file utils.py.

                :
  filled_bins=0.
  nbins=h.GetNbinsX()
  for ibin in xrange(nbins):
    if h.GetBinContent(ibin)>0:
      filled_bins+=1
  #print "%s %s --> %s" %(filled_bins,nbins,filled_bins/nbins)
  if filled_bins/nbins < .5:
    return True
  else:
    return False

#-------------------------------------------------------------------------------

def utils::literal2root	(		literal,
			rootType
	)

Definition at line 69 of file utils.py.

                                   :
  bitsarray = array.array('B')
  bitsarray.fromstring(literal.decode('hex'))

  tbuffer=0
  try:  
      tbuffer = TBufferFile(TBufferFile.kRead, len(bitsarray), bitsarray, False,0)
  except:
      print "could not transform to object array:"
      print [ i for i in  bitsarray ]
  
  # replace a couple of shortcuts with the real root class name
  if rootType == 'TPROF':
      rootType = 'TProfile'
  if rootType == 'TPROF2D':
      rootType = 'TProfile2D'
  
  root_class=eval(rootType+'.Class()')
  
  return tbuffer.ReadObject(root_class)
  
#-------------------------------------------------------------------------------

def utils::logger	(		msg_level,
			message
	)

Definition at line 47 of file utils.py.

00048                              :
00049   if msg_level>=_log_level:
00050     print "[%s] %s" %(asctime(),message)
00051 
#-------------------------------------------------------------------------------

def utils::make_files_pairs	(		files,
			verbose = True
	)

Definition at line 552 of file utils.py.

                                         :
    ## Select functions to use
    if is_relvaldata(files):
        is_relval_data = True
        get_cmssw_version = get_relvaldata_cmssw_version
        get_id = get_relvaldata_id
        get_max_version = get_relvaldata_max_version
        # print 'Pairing Data RelVal files.'
    else:
        is_relval_data = False
        get_cmssw_version = get_relval_cmssw_version
        get_id = get_relval_id
        get_max_version = get_relval_max_version
        # print 'Pairing Monte Carlo RelVal files.'

    ## Divide files into groups
    versions_files = dict()
    for file in files:
        version = get_cmssw_version(file)
        if versions_files.has_key(version):
            versions_files[version].append(file)
        else:
            versions_files[version] = [file]

    ## Print the division into groups
    if verbose:
        print '\nFound versions:'
        for version in versions_files:
            print '%s: %d files' % (str(version),  len(versions_files[version]))

    if len(versions_files.keys()) <= 1:
        print '\nFound too little versions, there is nothing to pair. Exiting...\n'
        exit()

    ## Select two biggest groups.
    versions = versions_files.keys()
    sizes = [len(value) for value in versions_files.values()]
    v1 = versions[sizes.index(max(sizes))]
    versions.remove(v1)
    sizes.remove(max(sizes))
    v2 = versions[sizes.index(max(sizes))]

    ## Print two biggest groups.
    if verbose:
        print '\nPairing %s (%d files) and %s (%d files)' % (str(v1),
                len(versions_files[v1]), str(v2), len(versions_files[v2]))

    ## Pairing two versions
    print '\nGot pairs:'
    pairs = []
    for unique_id in set([get_id(file) for file in versions_files[v1]]):
        if is_relval_data:
            dataset_re = re.compile(unique_id[0]+'_')
            run_re = re.compile(unique_id[1])
            c1_files = [file for file in versions_files[v1] if dataset_re.search(file) and run_re.search(file)]
            c2_files = [file for file in versions_files[v2] if dataset_re.search(file) and run_re.search(file)]
        else:
            dataset_re = re.compile(unique_id+'_')
            c1_files = [file for file in versions_files[v1] if dataset_re.search(file)]
            c2_files = [file for file in versions_files[v2] if dataset_re.search(file)]

        if len(c1_files) > 0 and len(c2_files) > 0:
            first_file = get_max_version(c1_files)
            second_file = get_max_version(c2_files)
            print '%s\n%s\n' % (first_file, second_file)
            pairs.extend((first_file, second_file))
    if verbose:
        print "Paired and got %d files.\n" % len(pairs)
    return pairs

def utils::parse_word

(

word

)

Definition at line 21 of file utils.py.

                    :
    "Parse word which contas double underscore tag"
    output = set()
    words  = word.split()
    for idx in xrange(0, len(words)):
        pat = words[idx]
        if  pat and len(pat) > 4 and pat[:2] == '__': # we found enclosure
            tag = pat[2:pat.rfind('__')]
            if  tag.find('__') != -1: # another pattern
                for item in tag.split('__'):
                    if  TAG.match(item):
                        output.add('__%s__' % item)
            else:
                output.add('__%s__' % tag)
    return output

def utils::profile2histo

(

profile

)

Definition at line 220 of file utils.py.

                          :
  if not profile.InheritsFrom("TH1"):
    return profile
    
  bin_low_edges=[]
  n_bins=profile.GetNbinsX()
  
  for ibin in xrange(1,n_bins+2):
    bin_low_edges.append(profile.GetBinLowEdge(ibin))
  bin_low_edges=array.array('f',bin_low_edges)
  histo=TH1F(profile.GetName(),profile.GetTitle(),n_bins,bin_low_edges)
  for ibin in xrange(0,n_bins+1):
    histo.SetBinContent(ibin,profile.GetBinContent(ibin))
    histo.SetBinError(ibin,profile.GetBinError(ibin))    
  
  return histo
#-------------------------------------------------------------------------------

def utils::setTDRStyle

(

)

Definition at line 52 of file utils.py.

References plotscripts::tdrStyle.

Referenced by FlavourHistograms< T >::plot().

                 :  
  this_dir=dirname(this_module_name)
  this_dir_one_up=this_dir[:this_dir.rfind("/")+1]
  #this_dir_two_up=this_dir_one_up[:this_dir_one_up.rfind("/")+1]
  style_file=''
  if os.environ.has_key("RELMON_SA"):
    style_file=this_dir_one_up+"data/tdrstyle_mod.C"
  else:
    style_file="%s/src/Utilities/RelMon/data/tdrstyle_mod.C"%(os.environ["CMSSW_BASE"])
  try:
    gROOT.ProcessLine(".L %s" %style_file)
    gROOT.ProcessLine("setTDRStyle()")
  except:
    "Print could not set the TDR style. File %s not found?" %style_file
    

#-------------------------------------------------------------------------------

def utils::test_env	(		tdir,
			tmpl
	)

Test user environment, look-up if user has run cmsenv, otherwise
provide meaningful error message back to the user.

Definition at line 37 of file utils.py.

                        :
    """
    Test user environment, look-up if user has run cmsenv, otherwise
    provide meaningful error message back to the user.
    """
    if  not tdir or not os.path.isdir(tdir):
        print "Unable to access template dir: %s" % tdir
        sys.exit(1)
    if  not os.listdir(tdir):
        print "No template files found in template dir %s" % tdir
        sys.exit(0)
    if  not tmpl:
        msg  = "No template type is provided, "
        msg += "see available templates via --templates option"
        print msg
        sys.exit(1)

def utils::tree

(

idir

)

Definition at line 127 of file utils.py.

              :
    "Print directory content, similar to tree UNIX command"
    if  idir[-1] == '/':
        idir = idir[-1]
    dsep = ''
    fsep = ''
    dtot = -1 # we'll not count initial directory
    ftot = 0
    for root, dirs, files in os.walk(idir):
        dirs  = root.split('/')
        ndirs = len(dirs)
        if  ndirs > 1:
            dsep  = '|  '*(ndirs-1)
        print '%s%s/' % (dsep, dirs[-1])
        dtot += 1
        for fname in files:
            fsep = dsep + '|--'
            print '%s %s' % (fsep, fname)
            ftot += 1
    if  dtot == -1 or not dtot:
        dmsg = ''
    else:
        dmsg = '%s directories,' % dtot
    if  ftot:
        fmsg = '%s file' % ftot
        if  ftot > 1:
            fmsg += 's'
    else:
        fmsg = ''
    if  dmsg and fmsg:
        print "Total: %s %s" % (dmsg, fmsg)
    else:
        print "No directories/files in %s" % idir

def utils::user_info

(

ainput = None

)

Definition at line 96 of file utils.py.

Referenced by cond::userInfo().

                          :
    "Return user name and office location, based on UNIX finger"
    if  ainput:
        return ainput
    pwdstr = pwd.getpwnam(os.getlogin())
    author = pwdstr.pw_gecos
    if  author and isinstance(author, basestring):
        author = author.split(',')[0]
    return author

def utils::wget

(

url

)

Fetch the WHOLE file, not in bunches... To be optimised.

Definition at line 451 of file utils.py.

00452              :
00453   """ Fetch the WHOLE file, not in bunches... To be optimised.
00454   """
00455   opener=build_opener(X509CertOpen())  
00456   datareq = Request(url)
00457   datareq.add_header('authenticated_wget', "The ultimate wgetter")    
00458   bin_content=None
00459   try:
00460     filename=basename(url)  
00461     print "Checking existence of file %s on disk..."%filename
00462     if not isfile("./%s"%filename):      
00463       bin_content=opener.open(datareq).read()
00464     else:
00465       print "File %s exists, skipping.." %filename
00466   except ValueError:
00467     print "Error: Unknown url %s" %url
00468   
00469   if bin_content!=None:  
00470     ofile = open(filename, 'wb')
00471     ofile.write(bin_content)
00472     ofile.close()
00473 
#-------------------------------------------------------------------------------

---

Variable Documentation

int utils::_log_level = 10

Definition at line 46 of file utils.py.

tuple utils::TAG = re.compile(r'[a-zA-Z0-9]')

Definition at line 19 of file utils.py.

utils::theargv = sys.argv

Definition at line 24 of file utils.py.