tools Namespace Reference

Functions
def	check_proxy ()
def	compute_product_string (product_string)
def	create_single_iov_db (inputs, run_number, output_db)
def	get_iovs (db, tag)
def	get_process_object (cfg)
def	get_tags (global_tag, records)
def	getDatasetStr (datasetpath)
def	getTerminalSize ()
def	haddLocal (localdir, result_file, extension='root')
def	interrupt (signum, frame)
def	listFilesLocal (paths, extension='.root')
def	loadCmsProcess (psetPath)
def	loadCmsProcessFile (psetName)
def	make_unique_runranges (ali_producer)
def	prependPaths (process, seqname)
def	remove_existing_object (path)
def	replace_factors (product_string, name, value)
def	replaceTemplate (template, opts)
def	run_checked (cmd, suppress_stderr=False)
def	stdinWait (text, default, time, timeoutDisplay=None, kwargs)

Function Documentation

check_proxy()

def tools.check_proxy

(

)

Check if GRID proxy has been initialized.

Definition at line 220 of file tools.py.

def check_proxy():
    """Check if GRID proxy has been initialized."""

    try:
        with open(os.devnull, "w") as dump:
            subprocess.check_call(["voms-proxy-info", "--exists"],
                                  stdout = dump, stderr = dump)
    except subprocess.CalledProcessError:
        return False
    return True

compute_product_string()

def tools.compute_product_string

(

product_string

)

Takes `product_string` and returns the product of the factors as string.

Arguments:
- `product_string`: string containing product ('<factor>*<factor>*...')

Definition at line 209 of file tools.py.

References nano_mu_digi_cff.float, conifer.reduce(), and str.

def compute_product_string(product_string):
    """Takes `product_string` and returns the product of the factors as string.

    Arguments:
    - `product_string`: string containing product ('<factor>*<factor>*...')
    """

    factors = [float(f) for f in product_string.split("*")]
    return str(reduce(lambda x,y: x*y, factors))

create_single_iov_db()

def tools.create_single_iov_db	(		inputs,
			run_number,
			output_db
	)

Create an sqlite file with single-IOV tags for alignment payloads.

Arguments:
- `inputs`: dictionary with input needed for payload extraction
- `run_number`: run for which the IOVs are selected
- `output_db`: name of the output sqlite file

Definition at line 14 of file tools.py.

References join(), print(), remove_existing_object(), run_checked(), and str.

def create_single_iov_db(inputs, run_number, output_db):
    """Create an sqlite file with single-IOV tags for alignment payloads.

    Arguments:
    - `inputs`: dictionary with input needed for payload extraction
    - `run_number`: run for which the IOVs are selected
    - `output_db`: name of the output sqlite file
    """

    # find the IOV containing `run_number`
    for record,tag in inputs.items():
        run_is_covered = False
        for iov in reversed(tag["iovs"]):
            if iov <= run_number:
                tag["since"] = str(iov)
                run_is_covered = True
                break
        if not run_is_covered:
            msg = ("Run number {0:d} is not covered in '{1:s}' ({2:s}) from"
                   " '{3:s}'.".format(run_number, tag["tag"], record,
                                      global_tag))
            print(msg)
            print("Aborting...")
            sys.exit(1)

    result = {}
    remove_existing_object(output_db)

    for record,tag in inputs.items():
        result[record] = {"connect": "sqlite_file:"+output_db,
                          "tag": "_".join([tag["tag"], tag["since"]])}

        if tag["connect"] == "pro":
            source_connect = "frontier://FrontierProd/CMS_CONDITIONS"
        elif tag["connect"] == "dev":
            source_connect = "frontier://FrontierPrep/CMS_CONDITIONS"
        else:
            source_connect = tag["connect"]

        cmd = ("conddb_import",
               "-f", source_connect,
               "-c", result[record]["connect"],
               "-i", tag["tag"],
               "-t", result[record]["tag"],
               "-b", str(run_number),
               "-e", str(run_number))
        run_checked(cmd)
    if len(inputs) > 0:
        run_checked(["sqlite3", output_db, "update iov set since=1"])

    return result

get_iovs()

def tools.get_iovs	(		db,
			tag
	)

Retrieve the list of IOVs from `db` for `tag`.

Arguments:
- `db`: database connection string
- `tag`: tag of database record

Definition at line 167 of file tools.py.

References python.cmstools.all(), ALCARECOTkAlBeamHalo_cff.filter, createfilelist.int, print(), and python.rootplot.root2matplotlib.replace().

def get_iovs(db, tag):
    """Retrieve the list of IOVs from `db` for `tag`.

    Arguments:
    - `db`: database connection string
    - `tag`: tag of database record
    """

    db = db.replace("sqlite_file:", "").replace("sqlite:", "")
    db = db.replace("frontier://FrontierProd/CMS_CONDITIONS", "pro")
    db = db.replace("frontier://FrontierPrep/CMS_CONDITIONS", "dev")

    con = conddb.connect(url = conddb.make_url(db))
    session = con.session()
    IOV = session.get_dbtype(conddb.IOV)

    iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all())
    if len(iovs) == 0:
        print("No IOVs found for tag '"+tag+"' in database '"+db+"'.")
        sys.exit(1)

    session.close()

    return sorted([int(item[0]) for item in iovs])

get_process_object()

def tools.get_process_object

(

cfg

)

Returns cms.Process object defined in `cfg`.

Arguments:
- `cfg`: path to CMSSW config file

Definition at line 87 of file tools.py.

References print().

def get_process_object(cfg):
    """Returns cms.Process object defined in `cfg`.

    Arguments:
    - `cfg`: path to CMSSW config file
    """

    sys.path.append(os.path.dirname(cfg)) # add location to python path
    cache_stdout = sys.stdout
    sys.stdout = open(os.devnull, "w")    # suppress unwanted output
    try:
        __configuration = \
            importlib.import_module(os.path.splitext(os.path.basename(cfg))[0])
    except Exception as e:
        print("Problem detected in configuration file '{0}'.".format(cfg))
        raise e
    sys.stdout = cache_stdout
    sys.path.pop()                        # clean up python path again
    try:
        os.remove(cfg+"c")                # try to remove temporary .pyc file
    except OSError as e:
        if e.args == (2, "No such file or directory"): pass
        else: raise

    return __configuration.process

get_tags()

def tools.get_tags	(		global_tag,
			records
	)

Get tags for `records` contained in `global_tag`.

Arguments:
- `global_tag`: global tag of interest
- `records`: database records of interest

Definition at line 132 of file tools.py.

References python.cmstools.all(), ALCARECOTkAlBeamHalo_cff.filter, and print().

def get_tags(global_tag, records):
    """Get tags for `records` contained in `global_tag`.

    Arguments:
    - `global_tag`: global tag of interest
    - `records`: database records of interest
    """

    if len(records) == 0: return {} # avoid useless DB query

    # check for auto GT
    if global_tag.startswith("auto:"):
        import Configuration.AlCa.autoCond as AC
        try:
            global_tag = AC.autoCond[global_tag.split("auto:")[-1]]
        except KeyError:
            print("Unsupported auto GT:", global_tag)
            sys.exit(1)

    # setting up the DB session
    con = conddb.connect(url = conddb.make_url())
    session = con.session()
    GlobalTagMap = session.get_dbtype(conddb.GlobalTagMap)

    # query tag names for records of interest contained in `global_tag`
    tags = session.query(GlobalTagMap.record, GlobalTagMap.tag_name).\
           filter(GlobalTagMap.global_tag_name == global_tag,
                  GlobalTagMap.record.in_(records)).all()

    # closing the DB session
    session.close()

    return {item[0]: {"tag": item[1], "connect": "pro"} for item in tags}

getDatasetStr()

def tools.getDatasetStr

(

datasetpath

)

Definition at line 20 of file tools.py.

def getDatasetStr(datasetpath):
    datasetstr = datasetpath
    datasetstr.strip()
    if datasetstr[0] == '/': datasetstr = datasetstr[1:]
    datasetstr = datasetstr.replace('/','_')

    return datasetstr

getTerminalSize()

def tools.getTerminalSize

(

)

Definition at line 96 of file tools.py.

References createfilelist.int.

Referenced by CrabHelper.CrabHelper.check_crabtask().

def getTerminalSize():
    #taken from http://stackoverflow.com/a/566752
    # returns width, size of terminal
    env = os.environ
    def ioctl_GWINSZ(fd):
        try:
            import fcntl, termios, struct, os
            cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ,
        '1234'))
        except:
            return
        return cr
    cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
    if not cr:
        try:
            fd = os.open(os.ctermid(), os.O_RDONLY)
            cr = ioctl_GWINSZ(fd)
            os.close(fd)
        except:
            pass
    if not cr:
        cr = (env.get('LINES', 25), env.get('COLUMNS', 80))

        
    return int(cr[1]), int(cr[0])

haddLocal()

def tools.haddLocal	(		localdir,
			result_file,
			extension = 'root'
	)

Definition at line 41 of file tools.py.

References listFilesLocal().

Referenced by DTWorkflow.DTWorkflow.prepare_common_write().

def haddLocal(localdir,result_file,extension = 'root'):
    if not os.path.exists( localdir ):
        raise ValueError("localdir for hadd operation does not exist" )

    files = listFilesLocal([localdir],extension)
    process = subprocess.Popen( ['hadd','-f', result_file] + files,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
    stdout = process.communicate()[0]
    return process.returncode

interrupt()

def tools.interrupt	(		signum,
			frame
	)

Definition at line 93 of file tools.py.

def interrupt(signum, frame):
    raise Exception("")

listFilesLocal()

def tools.listFilesLocal	(		paths,
			extension = '.root'
	)

Definition at line 28 of file tools.py.

Referenced by haddLocal().

def listFilesLocal(paths, extension = '.root'):
    file_paths = []
    for path in paths:
        if not os.path.exists( path ):
            log.error( "Specified input path '%s' does not exist!" % path )
            continue
        if path.endswith( extension ):
            file_paths.append( path )
        for root, dirnames, filenames in os.walk( path ):
            for filename in fnmatch.filter( filenames, '*' + extension ):
                file_paths.append( os.path.join( root, filename ) )
    return file_paths

loadCmsProcess()

def tools.loadCmsProcess

(

psetPath

)

Definition at line 56 of file tools.py.

Referenced by DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_submit(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_submit(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_submit(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_write().

def loadCmsProcess(psetPath):
    module = __import__(psetPath)
    process = sys.modules[psetPath].process

    import copy
    #FIXME: clone process
    #processNew = copy.deepcopy(process)
    processNew = copy.copy(process)
    return processNew

loadCmsProcessFile()

def tools.loadCmsProcessFile

(

psetName

)

Definition at line 52 of file tools.py.

def loadCmsProcessFile(psetName):
    pset = imp.load_source("psetmodule",psetName)
    return pset.process

make_unique_runranges()

def tools.make_unique_runranges

(

ali_producer

)

Derive unique run ranges from AlignmentProducer PSet.

Arguments:
- `ali_producer`: cms.PSet containing AlignmentProducer configuration

Definition at line 114 of file tools.py.

References createfilelist.int.

def make_unique_runranges(ali_producer):
    """Derive unique run ranges from AlignmentProducer PSet.

    Arguments:
    - `ali_producer`: cms.PSet containing AlignmentProducer configuration
    """

    if (hasattr(ali_producer, "RunRangeSelection") and
        len(ali_producer.RunRangeSelection) > 0):
        iovs = set([int(iov)
                    for sel in ali_producer.RunRangeSelection
                    for iov in sel.RunRanges])
        if len(iovs) == 0: return [1] # single IOV starting from run 1
        return sorted(iovs)
    else:
        return [1]                    # single IOV starting from run 1

prependPaths()

def tools.prependPaths	(		process,
			seqname
	)

Definition at line 66 of file tools.py.

Referenced by DTWorkflow.DTWorkflow.add_preselection(), and DTWorkflow.DTWorkflow.add_raw_option().

def prependPaths(process,seqname):
    for path in process.paths:
        getattr(process,path)._seq = getattr(process,seqname)*getattr(process,path)._seq

remove_existing_object()

def tools.remove_existing_object

(

path

)

Tries to remove file or directory located at `path`. If the user
has no delete permissions, the object is moved to a backup
file. If this fails it tries 5 times in total and then asks to
perform a cleanup by a user with delete permissions.

Arguments:
- `name`: name of the object to be (re)moved

Definition at line 232 of file tools.py.

References print(), and FastTimerService_cff.range.

Referenced by create_single_iov_db().

def remove_existing_object(path):
    """
    Tries to remove file or directory located at `path`. If the user
    has no delete permissions, the object is moved to a backup
    file. If this fails it tries 5 times in total and then asks to
    perform a cleanup by a user with delete permissions.

    Arguments:
    - `name`: name of the object to be (re)moved
    """

    if os.path.exists(path):
        remove_method = shutil.rmtree if os.path.isdir(path) else os.remove
        move_method = shutil.move if os.path.isdir(path) else os.rename
        try:
            remove_method(path)
        except OSError as e:
            if e.args != (13, "Permission denied"): raise
            backup_path = path.rstrip("/")+"~"
            for _ in range(5):
                try:
                    if os.path.exists(backup_path): remove_method(backup_path)
                    move_method(path, backup_path)
                    break
                except OSError as e:
                    if e.args != (13, "Permission denied"): raise
                    backup_path += "~"
        if os.path.exists(path):
            msg = ("Cannot remove '{}' due to missing 'delete' ".format(path)
                   +"permissions and the limit of 5 backups is reached. Please "
                   "ask a user with 'delete' permissions to clean up.")
            print(msg)
            sys.exit(1)

replace_factors()

def tools.replace_factors	(		product_string,
			name,
			value
	)

Takes a `product_string` and replaces all factors with `name` by `value`.

Arguments:
- `product_string`: input string containing a product
- `name`: name of the factor
- `value`: value of the factor

Definition at line 193 of file tools.py.

References str.

def replace_factors(product_string, name, value):
    """Takes a `product_string` and replaces all factors with `name` by `value`.

    Arguments:
    - `product_string`: input string containing a product
    - `name`: name of the factor
    - `value`: value of the factor
    """

    value = str(value)                                 # ensure it's a string
    return re.sub(r"^"+name+r"$", value,               # single factor
                  re.sub(r"[*]"+name+r"$", r"*"+value, # rhs
                         re.sub(r"^"+name+r"[*]", value+r"*", # lhs
                                re.sub(r"[*]"+name+r"[*]", r"*"+value+r"*",
                                       product_string))))

replaceTemplate()

def tools.replaceTemplate	(		template,
			opts
	)

Definition at line 10 of file tools.py.

References print(), fileinputsource_cfi.read, and str.

def replaceTemplate(template,**opts):
    result = open(template).read()
    for item in opts:
         old = '@@%s@@'%item
         new = str(opts[item])
         print("Replacing",old,"to",new)
         result = result.replace(old,new)

    return result

run_checked()

def tools.run_checked	(		cmd,
			suppress_stderr = False
	)

Run `cmd` and exit in case of failures.

Arguments:
- `cmd`: list containing the strings of the command
- `suppress_stderr`: suppress output from stderr

Definition at line 67 of file tools.py.

References join(), and print().

Referenced by create_single_iov_db().

def run_checked(cmd, suppress_stderr = False):
    """Run `cmd` and exit in case of failures.

    Arguments:
    - `cmd`: list containing the strings of the command
    - `suppress_stderr`: suppress output from stderr
    """

    try:
        with open(os.devnull, "w") as devnull:
            if suppress_stderr:
                subprocess.check_call(cmd, stdout = devnull, stderr = devnull)
            else:
                subprocess.check_call(cmd, stdout = devnull)
    except subprocess.CalledProcessError as e:
        print("Problem in running the following command:")
        print(" ".join(e.cmd))
        sys.exit(1)

stdinWait()

def tools.stdinWait	(		text,
			default,
			time,
			timeoutDisplay = None,
			kwargs
	)

Definition at line 70 of file tools.py.

References print().

Referenced by CrabHelper.CrabHelper.check_crabtask().

def stdinWait(text, default, time, timeoutDisplay = None, **kwargs):
    # taken and adjusted from http://stackoverflow.com/a/25860968
    signal.signal(signal.SIGALRM, interrupt)
    signal.alarm(time) # sets timeout
    global timeout
    try:
        inp = raw_input(text)
        signal.alarm(0)
        timeout = False
    except (KeyboardInterrupt):
        printInterrupt = kwargs.get("printInterrupt", True)
        if printInterrupt:
            print("Keyboard interrupt")
        timeout = True # Do this so you don't mistakenly get input when there is none
        inp = default
    except:
        timeout = True
        if not timeoutDisplay is None:
            print(timeoutDisplay)
        signal.alarm(0)
        inp = default
    return inp

---

Generated for CMSSW Reference Manual by  1.8.14