utils Namespace Reference

Classes
class	BinToBin
class	BinToBin1percent
class	Chi2
class	KS
class	StatisticalTest
class	unpickler

Functions
def	ask_ok (prompt, retries=4, complaint='yes or no')
def	code_generator (kwds)
def	friendly_since (time_type, since)
def	functor (code, kwds, debug=0)
def	get_relval_cmssw_version (file)
def	get_relval_id (file)
def	get_relval_max_version (files)
def	get_relval_version (file)
	-------------—— Make files pairs: RelVal utils ---------------—— More...
def	get_relvaldata_cmssw_version (file)
def	get_relvaldata_id (file)
	-----------—— Make files pairs: RelValData utils --------------—— More...
def	get_relvaldata_max_version (files)
def	get_relvaldata_version (file)
def	getNbins (h)
def	is_empty (h)
def	is_relvaldata (files)
	----------------------— Make files pairs -----------------------— More...
def	is_sparse (h)
def	literal2root (literal, rootType)
def	logger (msg_level, message)
def	make_files_pairs (files, verbose=True)
def	parse_word (word)
def	profile2histo (profile)
def	setTDRStyle ()
def	test_env (tdir, tmpl)
def	to_datetime (date_string)
def	to_timestamp (obj)
def	tree (idir)
def	user_info (ainput=None)
def	wget (url)

Variables
	_log_level
	argv
	gErrorIgnoreLevel
	theargv

Detailed Description

File that contains utility functions used by various modules, but that do not fit into any single module.

Function Documentation

def utils.ask_ok	(		prompt,
			retries = 4,
			complaint = 'yes or no'
	)

Definition at line 419 of file utils.py.

def ask_ok(prompt, retries=4, complaint='yes or no'):
    while True:
        ok = raw_input(prompt)
        if ok in ('y', 'ye', 'yes'):
            return True
        if ok in ('n', 'no'):
            return False
        retries = retries - 1
        if retries < 0:
            raise IOError('refusenik user')
        print complaint

#-------------------------------------------------------------------------------

def utils.code_generator

(

kwds

)

Code generator function, parse user arguments, load and
return appropriate template generator module.

Definition at line 106 of file utils.py.

References harvestTrackValidationPlots.str.

Referenced by cms.generate(), and main.generator().

def code_generator(kwds):
    """
    Code generator function, parse user arguments, load and
    return appropriate template generator module.
    """
    debug = kwds.get('debug', None)
    if  debug:
        print "Configuration:"
        pprint.pprint(kwds)
    try:
        klass  = kwds.get('tmpl')
        mname  = 'FWCore.Skeletons.%s' % klass.lower()
        module = __import__(mname, fromlist=[klass])
    except ImportError as err:
        klass  = 'AbstractPkg'
        module = __import__('FWCore.Skeletons.pkg', fromlist=[klass])
        if  debug:
            print "%s, will use %s" % (str(err), klass)
    obj = getattr(module, klass)(kwds)
    return obj

def utils.friendly_since	(		time_type,
			since
	)

Takes a since and, if it is Run-based expressed as Lumi-based,
returns the run number.
Otherwise, returns the since without transformations.

Definition at line 18 of file utils.py.

def friendly_since(time_type, since):
    """
    Takes a since and, if it is Run-based expressed as Lumi-based,
    returns the run number.
    Otherwise, returns the since without transformations.
    """
    if time_type == "Run" and (since & 0xffffff) == 0:
        return since >> 32
    else:
        return since

def utils.functor	(		code,
			kwds,
			debug = 0
	)

Auto-generate and execute function with given code and configuration
For details of compile/exec/eval see
http://lucumr.pocoo.org/2011/2/1/exec-in-python/

Definition at line 54 of file utils.py.

References join().

Referenced by pkg.AbstractPkg.write().

def functor(code, kwds, debug=0):
    """
    Auto-generate and execute function with given code and configuration
    For details of compile/exec/eval see
    http://lucumr.pocoo.org/2011/2/1/exec-in-python/
    """
    args  = []
    for key, val in kwds.items():
        if  isinstance(val, basestring):
            arg = '%s="%s"' % (key, val)
        elif isinstance(val, list):
            arg = '%s=%s' % (key, val)
        else:
            msg = 'Unsupported data type "%s" <%s>' % (val, type(val)) 
            raise Exception(msg)
        args.append(arg)
    func  = '\nimport sys'
    func += '\nimport StringIO'
    func += "\ndef func(%s):\n" % ','.join(args)
    func += code
    func += """
def capture():
    "Capture snippet printous"
    old_stdout = sys.stdout
    sys.stdout = StringIO.StringIO()
    func()
    out = sys.stdout.getvalue()
    sys.stdout = old_stdout
    return out\n
capture()\n"""
    if  debug:
        print "\n### generated code\n"
        print func
    # compile python code as exec statement
    obj   = compile(func, '<string>', 'exec')
    # define execution namespace
    namespace = {}
    # execute compiled python code in given namespace
    exec(obj, namespace)
    # located generated function object, run it and return its results
    return namespace['capture']()

def utils.get_relval_cmssw_version

(

file

)

Definition at line 532 of file utils.py.

def get_relval_cmssw_version(file):
    cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
    gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-([\w\d]*)_V\d*\w?(_[\w\d]*)?-v', file)
    if cmssw_release and gr_r_version:
        if "PU" in gr_r_version[0][0] and not "FastSim" in file:
            __gt = re.sub('^[^_]*_', "", gr_r_version[0][0])
            __process_string = gr_r_version[0][1]
            return (__gt, __process_string)
        elif "PU" in gr_r_version[0][0] and "FastSim" in file:   #a check for FastSimPU samples
            return (cmssw_release[0], "PU_")                     #with possibly different GT's
        return (cmssw_release[0], gr_r_version[0])

def utils.get_relval_id

(

file

)

Returns unique relval ID (dataset name) for a given file.

Definition at line 544 of file utils.py.

References edmIntegrityCheck.group.

def get_relval_id(file):
    """Returns unique relval ID (dataset name) for a given file."""
    dataset_name = re.findall('R\d{9}__([\w\D]*)__CMSSW_', file)
    __process_string = re.search('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-([\w\d]*)_V\d*\w?(_[\w\d]*)?-v', file)
    _ps = ""
    if __process_string:
        if "PU" in __process_string.group(1) and not "FastSim" in file:
            _ps = re.search('^[^_]*_', __process_string.group(1)).group()
        elif "PU" in __process_string.group(1) and "FastSim" in file:
            return dataset_name[0]+"_", _ps ##some testing is needed
    return dataset_name[0], _ps

def utils.get_relval_max_version

(

files

)

Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 520 of file utils.py.

References get_relval_version().

def get_relval_max_version(files):
    """Returns file with maximum version at a) beggining of the file,
    e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
    max_file = files[0]
    max_v = get_relval_version(files[0])
    for file in files:
        file_v = get_relval_version(file)
        if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
            max_file = file
            max_v = file_v
    return max_file

def utils.get_relval_version

(

file

)

-------------—— Make files pairs: RelVal utils ---------------——

Returns tuple (CMSSW version, run version) for specified file.

Definition at line 513 of file utils.py.

References createfilelist.int.

Referenced by get_relval_max_version().

def get_relval_version(file):
    """Returns tuple (CMSSW version, run version) for specified file."""
    cmssw_version = re.findall('DQM_V(\d*)_', file)
    run_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-[\w\d]*_V\d*\w?(?:_[\w\d]*)?-v(\d*)__', file)
    if cmssw_version and run_version:
        return (int(cmssw_version[0]), int(run_version[0]))

def utils.get_relvaldata_cmssw_version

(

file

)

Returns tuple (CMSSW release, GR_R version) for specified RelValData file.

Definition at line 482 of file utils.py.

def get_relvaldata_cmssw_version(file):
    """Returns tuple (CMSSW release, GR_R version) for specified RelValData file."""
    cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
    gr_r_version = re.findall('-(GR_R_\d*_V\d*\w?)(?:_RelVal)?_', file)
    if not gr_r_version:
        gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-(\w*)_RelVal_', file)
    if cmssw_release and gr_r_version:
        return (cmssw_release[0], gr_r_version[0])

def utils.get_relvaldata_id

(

file

)

-----------—— Make files pairs: RelValData utils --------------——

Returns unique relvaldata ID for a given file.

Definition at line 472 of file utils.py.

def get_relvaldata_id(file):
    """Returns unique relvaldata ID for a given file."""
    run_id = re.search('R\d{9}', file)
    run = re.search('_RelVal_([\w\d]*)-v\d__', file)
    if not run:
        run = re.search('GR_R_\d*_V\d*C?_([\w\d]*)-v\d__', file)
    if run_id and run:
        return (run_id.group(), run.group(1))
    return None

def utils.get_relvaldata_max_version

(

files

)

Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 500 of file utils.py.

References get_relvaldata_version().

def get_relvaldata_max_version(files):
    """Returns file with maximum version at a) beggining of the file,
    e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
    max_file = files[0]
    max_v = get_relvaldata_version(files[0])
    for file in files:
        file_v = get_relvaldata_version(file)
        if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
            max_file = file
            max_v = file_v
    return max_file

def utils.get_relvaldata_version

(

file

)

Returns tuple (CMSSW version, run version) for specified file.

Definition at line 491 of file utils.py.

References createfilelist.int.

Referenced by get_relvaldata_max_version().

def get_relvaldata_version(file):
    """Returns tuple (CMSSW version, run version) for specified file."""
    cmssw_version = re.findall('DQM_V(\d*)_', file)
    run_version = re.findall('_RelVal_[\w\d]*-v(\d)__', file)
    if not run_version:
        run_version = re.findall('GR_R_\d*_V\d*C?_[\w\d]*-v(\d)__', file)
    if cmssw_version and run_version:
        return (int(cmssw_version[0]), int(run_version[0]))

def utils.getNbins

(

h

)

Definition at line 89 of file utils.py.

Referenced by utils.Chi2.absval(), utils.BinToBin.do_test(), utils.BinToBin1percent.do_test(), and utils.StatisticalTest.get_rank().

def getNbins(h):
  biny=h.GetNbinsY()
  if biny>1:biny+=1
  binz=h.GetNbinsZ()
  if binz>1:binz+=1
  return (h.GetNbinsX()+1)*(biny)*(binz)

#-------------------------------------------------------------------------------

def utils.is_empty

(

h

)

Definition at line 167 of file utils.py.

Referenced by DTSectColl.addTSTheta(), and utils.StatisticalTest.get_rank().

def is_empty(h):
  for i in xrange(1,getNbins(h)):
    if h.GetBinContent(i)!=0: return False
  return True
  #return h.GetSumOfWeights()==0

#-------------------------------------------------------------------------------

def utils.is_relvaldata

(

files

)

----------------------— Make files pairs -----------------------—

Definition at line 557 of file utils.py.

References any().

Referenced by make_files_pairs().

def is_relvaldata(files):
    is_relvaldata_re = re.compile('_RelVal_')
    return any([is_relvaldata_re.search(filename) for filename in files])

def utils.is_sparse

(

h

)

Definition at line 175 of file utils.py.

def is_sparse(h):
  filled_bins=0.
  nbins=h.GetNbinsX()
  for ibin in xrange(nbins):
    if h.GetBinContent(ibin)>0:
      filled_bins+=1
  #print "%s %s --> %s" %(filled_bins,nbins,filled_bins/nbins)
  if filled_bins/nbins < .5:
    return True
  else:
    return False

#-------------------------------------------------------------------------------

def utils.literal2root	(		literal,
			rootType
	)

Definition at line 66 of file utils.py.

def literal2root (literal,rootType):
  bitsarray = array.array('B')
  bitsarray.fromstring(literal.decode('hex'))

  tbuffer=0
  try:  
      tbuffer = TBufferFile(TBufferFile.kRead, len(bitsarray), bitsarray, False,0)
  except:
      print "could not transform to object array:"
      print [ i for i in  bitsarray ]
  
  # replace a couple of shortcuts with the real root class name
  if rootType == 'TPROF':
      rootType = 'TProfile'
  if rootType == 'TPROF2D':
      rootType = 'TProfile2D'
  
  root_class=eval(rootType+'.Class()')
  
  return tbuffer.ReadObject(root_class)
  
#-------------------------------------------------------------------------------

def utils.logger	(		msg_level,
			message
	)

Definition at line 44 of file utils.py.

def logger(msg_level,message):
  if msg_level>=_log_level:
    print "[%s] %s" %(asctime(),message)

#-------------------------------------------------------------------------------

def utils.make_files_pairs	(		files,
			verbose = True
	)

Definition at line 561 of file utils.py.

References mps_alisetup.append, cmsPerfStripChart.dict, cmsRelvalreport.exit, is_relvaldata(), hpstanc_transforms.max, and harvestTrackValidationPlots.str.

Referenced by ValidationMatrix.get_filenames_from_pool().

def make_files_pairs(files, verbose=True):
    ## Select functions to use
    if is_relvaldata(files):
        is_relval_data = True
        get_cmssw_version = get_relvaldata_cmssw_version
        get_id = get_relvaldata_id
        get_max_version = get_relvaldata_max_version
        # print 'Pairing Data RelVal files.'
    else:
        is_relval_data = False
        get_cmssw_version = get_relval_cmssw_version
        get_id = get_relval_id
        get_max_version = get_relval_max_version
        # print 'Pairing Monte Carlo RelVal files.'

    ## Divide files into groups
    versions_files = dict()
    for file in files:
        version = get_cmssw_version(file)
        if version in versions_files:
            versions_files[version].append(file)
        else:
            versions_files[version] = [file]

    ## Print the division into groups
    if verbose:
        print '\nFound versions:'
        for version in versions_files:
            print '%s: %d files' % (str(version),  len(versions_files[version]))

    if len(versions_files.keys()) <= 1:
        print '\nFound too little versions, there is nothing to pair. Exiting...\n'
        exit()

    ## Select two biggest groups.
    versions = versions_files.keys()
    sizes = [len(value) for value in versions_files.values()]
    v1 = versions[sizes.index(max(sizes))]
    versions.remove(v1)
    sizes.remove(max(sizes))
    v2 = versions[sizes.index(max(sizes))]

    ## Print two biggest groups.
    if verbose:
        print '\nPairing %s (%d files) and %s (%d files)' % (str(v1),
                len(versions_files[v1]), str(v2), len(versions_files[v2]))

    ## Pairing two versions
    print '\nGot pairs:'
    pairs = []
    for unique_id in set([get_id(file) for file in versions_files[v1]]):
        if is_relval_data:
            dataset_re = re.compile(unique_id[0]+'_')
            run_re = re.compile(unique_id[1])
            c1_files = [file for file in versions_files[v1] if dataset_re.search(file) and run_re.search(file)]
            c2_files = [file for file in versions_files[v2] if dataset_re.search(file) and run_re.search(file)]
        else:
            dataset_re = re.compile(unique_id[0]+'_')
            ps_re = re.compile(unique_id[1])
            ##compile a PU re and search also for same PU
            c1_files = [file for file in versions_files[v1] if dataset_re.search(file) and ps_re.search(file)]
            c2_files = [file for file in versions_files[v2] if dataset_re.search(file) and ps_re.search(file)]

        if len(c1_files) > 0 and len(c2_files) > 0:
            first_file = get_max_version(c1_files)
            second_file = get_max_version(c2_files)
            print '%s\n%s\n' % (first_file, second_file)
            pairs.extend((first_file, second_file))
    if verbose:
        print "Paired and got %d files.\n" % len(pairs)
    return pairs

def utils.parse_word

(

word

)

Definition at line 21 of file utils.py.

Referenced by pkg.AbstractPkg.tmpl_tags().

def parse_word(word):
    "Parse word which contas double underscore tag"
    output = set()
    words  = word.split()
    for idx in xrange(0, len(words)):
        pat = words[idx]
        if  pat and len(pat) > 4 and pat[:2] == '__': # we found enclosure
            tag = pat[2:pat.rfind('__')]
            if  tag.find('__') != -1: # another pattern
                for item in tag.split('__'):
                    if  TAG.match(item):
                        output.add('__%s__' % item)
            else:
                output.add('__%s__' % tag)
    return output

def utils.profile2histo

(

profile

)

Definition at line 216 of file utils.py.

def profile2histo(profile):
  if not profile.InheritsFrom("TH1"):
    return profile
    
  bin_low_edges=[]
  n_bins=profile.GetNbinsX()
  
  for ibin in xrange(1,n_bins+2):
    bin_low_edges.append(profile.GetBinLowEdge(ibin))
  bin_low_edges=array.array('f',bin_low_edges)
  histo=TH1F(profile.GetName(),profile.GetTitle(),n_bins,bin_low_edges)
  for ibin in xrange(0,n_bins+1):
    histo.SetBinContent(ibin,profile.GetBinContent(ibin))
    histo.SetBinError(ibin,profile.GetBinError(ibin))    
  
  return histo
#-------------------------------------------------------------------------------

def utils.setTDRStyle

(

)

Definition at line 49 of file utils.py.

References compare_using_db.dirname.

def setTDRStyle():  
  this_dir=dirname(this_module_name)
  this_dir_one_up=this_dir[:this_dir.rfind("/")+1]
  #this_dir_two_up=this_dir_one_up[:this_dir_one_up.rfind("/")+1]
  style_file=''
  if "RELMON_SA" in os.environ:
    style_file=this_dir_one_up+"data/tdrstyle_mod.C"
  else:
    style_file="%s/src/Utilities/RelMon/data/tdrstyle_mod.C"%(os.environ["CMSSW_BASE"])
  try:
    gROOT.ProcessLine(".L %s" %style_file)
    gROOT.ProcessLine("setTDRStyle()")
  except:
    "Print could not set the TDR style. File %s not found?" %style_file
    

#-------------------------------------------------------------------------------

def utils.test_env	(		tdir,
			tmpl
	)

Test user environment, look-up if user has run cmsenv, otherwise
provide meaningful error message back to the user.

Definition at line 37 of file utils.py.

Referenced by main.generator().

def test_env(tdir, tmpl):
    """
    Test user environment, look-up if user has run cmsenv, otherwise
    provide meaningful error message back to the user.
    """
    if  not tdir or not os.path.isdir(tdir):
        print "Unable to access template dir: %s" % tdir
        sys.exit(1)
    if  not os.listdir(tdir):
        print "No template files found in template dir %s" % tdir
        sys.exit(0)
    if  not tmpl:
        msg  = "No template type is provided, "
        msg += "see available templates via --templates option"
        print msg
        sys.exit(1)

def utils.to_datetime

(

date_string

)

Takes a date string with the format Y-m-d H:m:S.f and gives back a datetime.datetime object

Definition at line 12 of file utils.py.

Referenced by querying_tests.global_tag_tests().

def to_datetime(date_string):
    """
    Takes a date string with the format Y-m-d H:m:S.f and gives back a datetime.datetime object
    """
    return datetime.datetime.strptime(date_string.replace(",", "."), "%Y-%m-%d %H:%M:%S.%f")

def utils.to_timestamp

(

obj

)

Takes a datetime object and outputs a timestamp string with the format Y-m-d H:m:S.f

Definition at line 6 of file utils.py.

Referenced by uploads.uploader.filter_iovs_by_fcsr(), models.generate(), uploads.log(), and uploads.uploader.send_blob().

def to_timestamp(obj):
    """
    Takes a datetime object and outputs a timestamp string with the format Y-m-d H:m:S.f
    """
    return obj.strftime('%Y-%m-%d %H:%M:%S.%f') if isinstance(obj, datetime.datetime) else obj

def utils.tree

(

idir

)

Definition at line 127 of file utils.py.

def tree(idir):
    "Print directory content, similar to tree UNIX command"
    if  idir[-1] == '/':
        idir = idir[-1]
    dsep = ''
    fsep = ''
    dtot = -1 # we'll not count initial directory
    ftot = 0
    for root, dirs, files in os.walk(idir):
        dirs  = root.split('/')
        ndirs = len(dirs)
        if  ndirs > 1:
            dsep  = '|  '*(ndirs-1)
        print '%s%s/' % (dsep, dirs[-1])
        dtot += 1
        for fname in files:
            fsep = dsep + '|--'
            print '%s %s' % (fsep, fname)
            ftot += 1
    if  dtot == -1 or not dtot:
        dmsg = ''
    else:
        dmsg = '%s directories,' % dtot
    if  ftot:
        fmsg = '%s file' % ftot
        if  ftot > 1:
            fmsg += 's'
    else:
        fmsg = ''
    if  dmsg and fmsg:
        print "Total: %s %s" % (dmsg, fmsg)
    else:
        print "No directories/files in %s" % idir

def utils.user_info

(

ainput = None

)

Definition at line 96 of file utils.py.

def user_info(ainput=None):
    "Return user name and office location, based on UNIX finger"
    if  ainput:
        return ainput
    pwdstr = pwd.getpwnam(os.getlogin())
    author = pwdstr.pw_gecos
    if  author and isinstance(author, basestring):
        author = author.split(',')[0]
    return author

def utils.wget

(

url

)

Fetch the WHOLE file, not in bunches... To be optimised.

Definition at line 447 of file utils.py.

References estimatePileup.basename.

def wget(url):
  """ Fetch the WHOLE file, not in bunches... To be optimised.
  """
  opener=build_opener(X509CertOpen())  
  datareq = Request(url)
  datareq.add_header('authenticated_wget', "The ultimate wgetter")    
  bin_content=None
  try:
    filename=basename(url)  
    print "Checking existence of file %s on disk..."%filename
    if not isfile("./%s"%filename):      
      bin_content=opener.open(datareq).read()
    else:
      print "File %s exists, skipping.." %filename
  except ValueError:
    print "Error: Unknown url %s" %url
  
  if bin_content!=None:  
    ofile = open(filename, 'wb')
    ofile.write(bin_content)
    ofile.close()

#-------------------------------------------------------------------------------

Variable Documentation

utils._log_level

private

Definition at line 43 of file utils.py.

utils.argv

Definition at line 22 of file utils.py.

utils.gErrorIgnoreLevel

Definition at line 25 of file utils.py.

Referenced by main(), FWFileEntry.openFile(), TkAlStyle.set(), and topLevelPSet().

utils.theargv

Definition at line 21 of file utils.py.

---

Generated for CMSSW Reference Manual by  1.8.11