tools Namespace Reference

Functions
def	_test ()
def	check_proxy ()
def	compute_product_string (product_string)
def	create_single_iov_db (inputs, run_number, output_db)
def	get_iovs (db, tag)
def	get_process_object (cfg)
def	get_tags (global_tag, records)
def	getDatasetStr (datasetpath)
def	getTerminalSize ()
def	haddLocal (localdir, result_file, extension='root')
def	interrupt (signum, frame)
def	listFilesLocal (paths, extension='.root')
def	loadCmsProcess (psetPath)
def	loadCmsProcessFile (psetName)
def	make_unique_runranges (ali_producer)
def	prependPaths (process, seqname)
def	read_constant_graph (graph_path, create_session=None, as_text=None)
def	remove_existing_object (path)
def	replace_factors (product_string, name, value)
def	replaceTemplate (template, **opts)
def	run_checked (cmd, suppress_stderr=False)
def	stdinWait (text, default, time, timeoutDisplay=None, **kwargs)
def	visualize_graph (graph, log_dir=None, start_tensorboard=False, tensorboard_args="", **kwargs)
def	write_constant_graph (session, output_names, graph_path, **kwargs)

Variables
	__all__
	TF1
	tf1
	TF2

Function Documentation

_test()

def tools._test ( )

private

Internal test of the above functions based on the deepjet model.

Definition at line 193 of file tools.py.

def _test():
    """
    Internal test of the above functions based on the deepjet model.
    """
    deepjet_model = "/cvmfs/cms.cern.ch/slc7_amd64_gcc820/cms/data-RecoBTag-Combined/V01-02-01/" \
        "RecoBTag/Combined/data/DeepFlavourV03_10X_training/constant_graph.pb"
    deepjet_output = "ID_pred/Softmax"
 
    if not os.path.exists(deepjet_model):
        print("cannot run tests as deepjet model '{}' does not exist".format(deepjet_model))
        sys.exit(1)
 
    # load the graph
    read_constant_graph(deepjet_model, create_session=False)
    if tf1:
        g, s = read_constant_graph(deepjet_model, create_session=True)
 
    # write the graph
    if tf1:
        with tempfile.NamedTemporaryFile(suffix=".pb") as ntf:
            write_constant_graph(s, [deepjet_output], ntf.name)
 
    # visualize the graph
    if tf1:
        visualize_graph(g)
 
 

References print(), read_constant_graph(), visualize_graph(), and write_constant_graph().

check_proxy()

def tools.check_proxy

(

)

Check if GRID proxy has been initialized.

Definition at line 221 of file tools.py.

def check_proxy():
    """Check if GRID proxy has been initialized."""
 
    try:
        with open(os.devnull, "w") as dump:
            subprocess.check_call(["voms-proxy-info", "--exists"],
                                  stdout = dump, stderr = dump)
    except subprocess.CalledProcessError:
        return False
    return True
 
 

compute_product_string()

def tools.compute_product_string

(

product_string

)

Takes `product_string` and returns the product of the factors as string.

Arguments:
- `product_string`: string containing product ('<factor>*<factor>*...')

Definition at line 210 of file tools.py.

def compute_product_string(product_string):
    """Takes `product_string` and returns the product of the factors as string.
 
    Arguments:
    - `product_string`: string containing product ('<factor>*<factor>*...')
    """
 
    factors = [float(f) for f in product_string.split("*")]
    return str(reduce(lambda x,y: x*y, factors))
 
 

References dqmMemoryStats.float, and str.

create_single_iov_db()

def tools.create_single_iov_db	(		inputs,
			run_number,
			output_db
	)

Create an sqlite file with single-IOV tags for alignment payloads.

Arguments:
- `inputs`: dictionary with input needed for payload extraction
- `run_number`: run for which the IOVs are selected
- `output_db`: name of the output sqlite file

Definition at line 15 of file tools.py.

def create_single_iov_db(inputs, run_number, output_db):
    """Create an sqlite file with single-IOV tags for alignment payloads.
 
    Arguments:
    - `inputs`: dictionary with input needed for payload extraction
    - `run_number`: run for which the IOVs are selected
    - `output_db`: name of the output sqlite file
    """
 
    # find the IOV containing `run_number`
    for record,tag in six.iteritems(inputs):
        run_is_covered = False
        for iov in reversed(tag["iovs"]):
            if iov <= run_number:
                tag["since"] = str(iov)
                run_is_covered = True
                break
        if not run_is_covered:
            msg = ("Run number {0:d} is not covered in '{1:s}' ({2:s}) from"
                   " '{3:s}'.".format(run_number, tag["tag"], record,
                                      global_tag))
            print(msg)
            print("Aborting...")
            sys.exit(1)
 
    result = {}
    remove_existing_object(output_db)
 
    for record,tag in six.iteritems(inputs):
        result[record] = {"connect": "sqlite_file:"+output_db,
                          "tag": "_".join([tag["tag"], tag["since"]])}
 
        if tag["connect"] == "pro":
            source_connect = "frontier://FrontierProd/CMS_CONDITIONS"
        elif tag["connect"] == "dev":
            source_connect = "frontier://FrontierPrep/CMS_CONDITIONS"
        else:
            source_connect = tag["connect"]
 
        cmd = ("conddb_import",
               "-f", source_connect,
               "-c", result[record]["connect"],
               "-i", tag["tag"],
               "-t", result[record]["tag"],
               "-b", str(run_number),
               "-e", str(run_number))
        run_checked(cmd)
    if len(inputs) > 0:
        run_checked(["sqlite3", output_db, "update iov set since=1"])
 
    return result
 
 

References join(), print(), remove_existing_object(), run_checked(), and str.

get_iovs()

def tools.get_iovs	(		db,
			tag
	)

Retrieve the list of IOVs from `db` for `tag`.

Arguments:
- `db`: database connection string
- `tag`: tag of database record

Definition at line 168 of file tools.py.

def get_iovs(db, tag):
    """Retrieve the list of IOVs from `db` for `tag`.
 
    Arguments:
    - `db`: database connection string
    - `tag`: tag of database record
    """
 
    db = db.replace("sqlite_file:", "").replace("sqlite:", "")
    db = db.replace("frontier://FrontierProd/CMS_CONDITIONS", "pro")
    db = db.replace("frontier://FrontierPrep/CMS_CONDITIONS", "dev")
 
    con = conddb.connect(url = conddb.make_url(db))
    session = con.session()
    IOV = session.get_dbtype(conddb.IOV)
 
    iovs = set(session.query(IOV.since).filter(IOV.tag_name == tag).all())
    if len(iovs) == 0:
        print("No IOVs found for tag '"+tag+"' in database '"+db+"'.")
        sys.exit(1)
 
    session.close()
 
    return sorted([int(item[0]) for item in iovs])
 
 

References python.cmstools.all(), ALCARECOTkAlBeamHalo_cff.filter, createfilelist.int, print(), and python.rootplot.root2matplotlib.replace().

get_process_object()

def tools.get_process_object

(

cfg

)

Returns cms.Process object defined in `cfg`.

Arguments:
- `cfg`: path to CMSSW config file

Definition at line 88 of file tools.py.

def get_process_object(cfg):
    """Returns cms.Process object defined in `cfg`.
 
    Arguments:
    - `cfg`: path to CMSSW config file
    """
 
    sys.path.append(os.path.dirname(cfg)) # add location to python path
    cache_stdout = sys.stdout
    sys.stdout = open(os.devnull, "w")    # suppress unwanted output
    try:
        __configuration = \
            importlib.import_module(os.path.splitext(os.path.basename(cfg))[0])
    except Exception as e:
        print("Problem detected in configuration file '{0}'.".format(cfg))
        raise e
    sys.stdout = cache_stdout
    sys.path.pop()                        # clean up python path again
    try:
        os.remove(cfg+"c")                # try to remove temporary .pyc file
    except OSError as e:
        if e.args == (2, "No such file or directory"): pass
        else: raise
 
    return __configuration.process
 
 

References print().

get_tags()

def tools.get_tags	(		global_tag,
			records
	)

Get tags for `records` contained in `global_tag`.

Arguments:
- `global_tag`: global tag of interest
- `records`: database records of interest

Definition at line 133 of file tools.py.

def get_tags(global_tag, records):
    """Get tags for `records` contained in `global_tag`.
 
    Arguments:
    - `global_tag`: global tag of interest
    - `records`: database records of interest
    """
 
    if len(records) == 0: return {} # avoid useless DB query
 
    # check for auto GT
    if global_tag.startswith("auto:"):
        import Configuration.AlCa.autoCond as AC
        try:
            global_tag = AC.autoCond[global_tag.split("auto:")[-1]]
        except KeyError:
            print("Unsupported auto GT:", global_tag)
            sys.exit(1)
 
    # setting up the DB session
    con = conddb.connect(url = conddb.make_url())
    session = con.session()
    GlobalTagMap = session.get_dbtype(conddb.GlobalTagMap)
 
    # query tag names for records of interest contained in `global_tag`
    tags = session.query(GlobalTagMap.record, GlobalTagMap.tag_name).\
           filter(GlobalTagMap.global_tag_name == global_tag,
                  GlobalTagMap.record.in_(records)).all()
 
    # closing the DB session
    session.close()
 
    return {item[0]: {"tag": item[1], "connect": "pro"} for item in tags}
 
 

References python.cmstools.all(), ALCARECOTkAlBeamHalo_cff.filter, and print().

getDatasetStr()

def tools.getDatasetStr

(

datasetpath

)

Definition at line 20 of file tools.py.

def getDatasetStr(datasetpath):
    datasetstr = datasetpath
    datasetstr.strip()
    if datasetstr[0] == '/': datasetstr = datasetstr[1:]
    datasetstr = datasetstr.replace('/','_')
 
    return datasetstr
 

getTerminalSize()

def tools.getTerminalSize

(

)

Definition at line 96 of file tools.py.

def getTerminalSize():
    #taken from http://stackoverflow.com/a/566752
    # returns width, size of terminal
    env = os.environ
    def ioctl_GWINSZ(fd):
        try:
            import fcntl, termios, struct, os
            cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ,
        '1234'))
        except:
            return
        return cr
    cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
    if not cr:
        try:
            fd = os.open(os.ctermid(), os.O_RDONLY)
            cr = ioctl_GWINSZ(fd)
            os.close(fd)
        except:
            pass
    if not cr:
        cr = (env.get('LINES', 25), env.get('COLUMNS', 80))
 
        
    return int(cr[1]), int(cr[0])

References createfilelist.int.

Referenced by CrabHelper.CrabHelper.check_crabtask().

haddLocal()

def tools.haddLocal	(		localdir,
			result_file,
			extension = 'root'
	)

Definition at line 41 of file tools.py.

def haddLocal(localdir,result_file,extension = 'root'):
    if not os.path.exists( localdir ):
        raise ValueError("localdir for hadd operation does not exist" )
 
    files = listFilesLocal([localdir],extension)
    process = subprocess.Popen( ['hadd','-f', result_file] + files,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
    stdout = process.communicate()[0]
    return process.returncode
 

References listFilesLocal().

Referenced by DTWorkflow.DTWorkflow.prepare_common_write().

interrupt()

def tools.interrupt	(		signum,
			frame
	)

Definition at line 93 of file tools.py.

def interrupt(signum, frame):
    raise Exception("")
 

listFilesLocal()

def tools.listFilesLocal	(		paths,
			extension = '.root'
	)

Definition at line 28 of file tools.py.

def listFilesLocal(paths, extension = '.root'):
    file_paths = []
    for path in paths:
        if not os.path.exists( path ):
            log.error( "Specified input path '%s' does not exist!" % path )
            continue
        if path.endswith( extension ):
            file_paths.append( path )
        for root, dirnames, filenames in os.walk( path ):
            for filename in fnmatch.filter( filenames, '*' + extension ):
                file_paths.append( os.path.join( root, filename ) )
    return file_paths
 

Referenced by haddLocal().

loadCmsProcess()

def tools.loadCmsProcess

(

psetPath

)

Definition at line 56 of file tools.py.

def loadCmsProcess(psetPath):
    module = __import__(psetPath)
    process = sys.modules[psetPath].process
 
    import copy
    #FIXME: clone process
    #processNew = copy.deepcopy(process)
    processNew = copy.copy(process)
    return processNew
 

Referenced by DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_submit(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_submit(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_submit(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_write().

loadCmsProcessFile()

def tools.loadCmsProcessFile

(

psetName

)

Definition at line 52 of file tools.py.

def loadCmsProcessFile(psetName):
    pset = imp.load_source("psetmodule",psetName)
    return pset.process
 

make_unique_runranges()

def tools.make_unique_runranges

(

ali_producer

)

Derive unique run ranges from AlignmentProducer PSet.

Arguments:
- `ali_producer`: cms.PSet containing AlignmentProducer configuration

Definition at line 115 of file tools.py.

def make_unique_runranges(ali_producer):
    """Derive unique run ranges from AlignmentProducer PSet.
 
    Arguments:
    - `ali_producer`: cms.PSet containing AlignmentProducer configuration
    """
 
    if (hasattr(ali_producer, "RunRangeSelection") and
        len(ali_producer.RunRangeSelection) > 0):
        iovs = set([int(iov)
                    for sel in ali_producer.RunRangeSelection
                    for iov in sel.RunRanges])
        if len(iovs) == 0: return [1] # single IOV starting from run 1
        return sorted(iovs)
    else:
        return [1]                    # single IOV starting from run 1
 
 

References createfilelist.int.

prependPaths()

def tools.prependPaths	(		process,
			seqname
	)

Definition at line 66 of file tools.py.

def prependPaths(process,seqname):
    for path in process.paths:
        getattr(process,path)._seq = getattr(process,seqname)*getattr(process,path)._seq
 

Referenced by DTWorkflow.DTWorkflow.add_preselection(), and DTWorkflow.DTWorkflow.add_raw_option().

read_constant_graph()

def tools.read_constant_graph	(		graph_path,
			create_session = None,
			as_text = None
	)

Reads a saved TensorFlow graph from *graph_path* and returns it. When *create_session* is
*True*, a session object (compatible with the v1 API) is created and returned as well as the
second value of a 2-tuple. The default value of *create_session* is *True* when TensorFlow v1
is detected, and *False* otherwise. When *as_text* is *True*, or *None* and the file extension
is ``".pbtxt"`` or ``".pb.txt"``, the content of the file at *graph_path* is expected to be a
human-readable text file. Otherwise, it is expected to be a binary protobuf file. Example:

.. code-block:: python

    graph = read_constant_graph("path/to/model.pb", create_session=False)

    graph, session = read_constant_graph("path/to/model.pb", create_session=True)

Definition at line 38 of file tools.py.

def read_constant_graph(graph_path, create_session=None, as_text=None):
    """
    Reads a saved TensorFlow graph from *graph_path* and returns it. When *create_session* is
    *True*, a session object (compatible with the v1 API) is created and returned as well as the
    second value of a 2-tuple. The default value of *create_session* is *True* when TensorFlow v1
    is detected, and *False* otherwise. When *as_text* is *True*, or *None* and the file extension
    is ``".pbtxt"`` or ``".pb.txt"``, the content of the file at *graph_path* is expected to be a
    human-readable text file. Otherwise, it is expected to be a binary protobuf file. Example:
 
    .. code-block:: python
 
        graph = read_constant_graph("path/to/model.pb", create_session=False)
 
        graph, session = read_constant_graph("path/to/model.pb", create_session=True)
    """
    if as_text is None:
        as_text = graph_path.endswith((".pbtxt", ".pb.txt"))
 
    graph = tf.Graph()
    with graph.as_default():
        graph_def = graph.as_graph_def()
 
        if as_text:
            # use a simple pb reader to load the file into graph_def
            from google.protobuf import text_format
            with open(graph_path, "r") as f:
                text_format.Merge(f.read(), graph_def)
 
        else:
            # use the gfile api depending on the TF version
            if TF1:
                from tensorflow.python.platform import gfile
                with gfile.FastGFile(graph_path, "rb") as f:
                    graph_def.ParseFromString(f.read())
            else:
                with tf.io.gfile.GFile(graph_path, "rb") as f:
                    graph_def.ParseFromString(f.read())
 
        # import the graph_def (pb object) into the actual graph
        tf.import_graph_def(graph_def, name="")
 
    # determine the create_session default
    if create_session is None:
        create_session = TF1
 
    if create_session:
        if not tf1:
            raise NotImplementedError("the v1 compatibility layer of TensorFlow v2 (tf.compat.v1) "
                "is required by read_constant_graph when create_session is True, but missing")
        session = tf1.Session(graph=graph)
        return graph, session
    else:
        return graph
 
 

Referenced by _test(), and visualize_graph().

remove_existing_object()

def tools.remove_existing_object

(

path

)

Tries to remove file or directory located at `path`. If the user
has no delete permissions, the object is moved to a backup
file. If this fails it tries 5 times in total and then asks to
perform a cleanup by a user with delete permissions.

Arguments:
- `name`: name of the object to be (re)moved

Definition at line 233 of file tools.py.

def remove_existing_object(path):
    """
    Tries to remove file or directory located at `path`. If the user
    has no delete permissions, the object is moved to a backup
    file. If this fails it tries 5 times in total and then asks to
    perform a cleanup by a user with delete permissions.
 
    Arguments:
    - `name`: name of the object to be (re)moved
    """
 
    if os.path.exists(path):
        remove_method = shutil.rmtree if os.path.isdir(path) else os.remove
        move_method = shutil.move if os.path.isdir(path) else os.rename
        try:
            remove_method(path)
        except OSError as e:
            if e.args != (13, "Permission denied"): raise
            backup_path = path.rstrip("/")+"~"
            for _ in range(5):
                try:
                    if os.path.exists(backup_path): remove_method(backup_path)
                    move_method(path, backup_path)
                    break
                except OSError as e:
                    if e.args != (13, "Permission denied"): raise
                    backup_path += "~"
        if os.path.exists(path):
            msg = ("Cannot remove '{}' due to missing 'delete' ".format(path)
                   +"permissions and the limit of 5 backups is reached. Please "
                   "ask a user with 'delete' permissions to clean up.")
            print(msg)
            sys.exit(1)

References print(), and FastTimerService_cff.range.

Referenced by create_single_iov_db().

replace_factors()

def tools.replace_factors	(		product_string,
			name,
			value
	)

Takes a `product_string` and replaces all factors with `name` by `value`.

Arguments:
- `product_string`: input string containing a product
- `name`: name of the factor
- `value`: value of the factor

Definition at line 194 of file tools.py.

def replace_factors(product_string, name, value):
    """Takes a `product_string` and replaces all factors with `name` by `value`.
 
    Arguments:
    - `product_string`: input string containing a product
    - `name`: name of the factor
    - `value`: value of the factor
    """
 
    value = str(value)                                 # ensure it's a string
    return re.sub(r"^"+name+r"$", value,               # single factor
                  re.sub(r"[*]"+name+r"$", r"*"+value, # rhs
                         re.sub(r"^"+name+r"[*]", value+r"*", # lhs
                                re.sub(r"[*]"+name+r"[*]", r"*"+value+r"*",
                                       product_string))))
 

References str.

replaceTemplate()

def tools.replaceTemplate	(		template,
		**	opts
	)

Definition at line 10 of file tools.py.

def replaceTemplate(template,**opts):
    result = open(template).read()
    for item in opts:
         old = '@@%s@@'%item
         new = str(opts[item])
         print("Replacing",old,"to",new)
         result = result.replace(old,new)
 
    return result
 

References print(), readEcalDQMStatus.read, and str.

run_checked()

def tools.run_checked	(		cmd,
			suppress_stderr = False
	)

Run `cmd` and exit in case of failures.

Arguments:
- `cmd`: list containing the strings of the command
- `suppress_stderr`: suppress output from stderr

Definition at line 68 of file tools.py.

def run_checked(cmd, suppress_stderr = False):
    """Run `cmd` and exit in case of failures.
 
    Arguments:
    - `cmd`: list containing the strings of the command
    - `suppress_stderr`: suppress output from stderr
    """
 
    try:
        with open(os.devnull, "w") as devnull:
            if suppress_stderr:
                subprocess.check_call(cmd, stdout = devnull, stderr = devnull)
            else:
                subprocess.check_call(cmd, stdout = devnull)
    except subprocess.CalledProcessError as e:
        print("Problem in running the following command:")
        print(" ".join(e.cmd))
        sys.exit(1)
 
 

References join(), and print().

Referenced by create_single_iov_db().

stdinWait()

def tools.stdinWait	(		text,
			default,
			time,
			timeoutDisplay = None,
		**	kwargs
	)

Definition at line 70 of file tools.py.

def stdinWait(text, default, time, timeoutDisplay = None, **kwargs):
    # taken and adjusted from http://stackoverflow.com/a/25860968
    signal.signal(signal.SIGALRM, interrupt)
    signal.alarm(time) # sets timeout
    global timeout
    try:
        inp = raw_input(text)
        signal.alarm(0)
        timeout = False
    except (KeyboardInterrupt):
        printInterrupt = kwargs.get("printInterrupt", True)
        if printInterrupt:
            print("Keyboard interrupt")
        timeout = True # Do this so you don't mistakenly get input when there is none
        inp = default
    except:
        timeout = True
        if not timeoutDisplay is None:
            print(timeoutDisplay)
        signal.alarm(0)
        inp = default
    return inp
 

References print().

Referenced by CrabHelper.CrabHelper.check_crabtask().

visualize_graph()

def tools.visualize_graph	(		graph,
			log_dir = None,
			start_tensorboard = False,
			tensorboard_args = "",
		**	kwargs
	)

Visualizes a TensorFlow *graph* by adding it to a ``tf.summary.FileWriter``. *graph* can be
either a graph object or a path to a pb file. In the latter case, :py:func:`read_constant_graph`
is used and all *kwargs* are forwarded. The file writer object is instantiated with a *log_dir*
which, when empty, defaults to a temporary directory. This is especially usefull when
*start_tensorboard* is *True*, in which case a subprocesses is started to run a *tensorboard*
instance with additional arguments given as a string *tensorboard_args*. The subprocess is
terminated on keyboard interrupt.

.. note::

    When used with TensorFlow v2, this function requires the v1 API compatibility layer. When
    :py:attr:`tf.compat.v1` is not available, a *NotImplementedError* is raised.

Definition at line 132 of file tools.py.

def visualize_graph(graph, log_dir=None, start_tensorboard=False, tensorboard_args="", **kwargs):
    """
    Visualizes a TensorFlow *graph* by adding it to a ``tf.summary.FileWriter``. *graph* can be
    either a graph object or a path to a pb file. In the latter case, :py:func:`read_constant_graph`
    is used and all *kwargs* are forwarded. The file writer object is instantiated with a *log_dir*
    which, when empty, defaults to a temporary directory. This is especially usefull when
    *start_tensorboard* is *True*, in which case a subprocesses is started to run a *tensorboard*
    instance with additional arguments given as a string *tensorboard_args*. The subprocess is
    terminated on keyboard interrupt.
 
    .. note::
 
        When used with TensorFlow v2, this function requires the v1 API compatibility layer. When
        :py:attr:`tf.compat.v1` is not available, a *NotImplementedError* is raised.
    """
    # complain when the v1 compatibility layer is not existing
    if not tf1:
        raise NotImplementedError("the v1 compatibility layer of TensorFlow v2 (tf.compat.v1) is "
            "required by visualize_graph, but missing")
 
    # prepare the log_dir
    is_tmp = not log_dir
    if is_tmp:
        log_dir = tempfile.mkdtemp()
    elif not os.path.exists(log_dir):
        os.makedirs(log_dir)
 
    # read the graph when a string is passed
    if isinstance(graph, six.string_types):
        graph = read_constant_graph(graph, create_session=False, **kwargs)
 
    # switch to non-eager mode for the FileWriter to work
    eager = getattr(tf1, "executing_eagerly", lambda: False)()
    if eager:
        tf1.disable_eager_execution()
 
    # write to file
    writer = tf1.summary.FileWriter(log_dir)
    writer.add_graph(graph)
 
    # reset the eager mode
    if eager:
        tf1.enable_eager_execution()
 
    # optionally start a tensorboard process
    if start_tensorboard:
        print("starting tensorboard with logdir {}".format(log_dir))
        cmd = "tensorboard --logdir '{}' {}".format(log_dir, tensorboard_args)
        p = subprocess.Popen(cmd, shell=True, executable="/bin/bash", preexec_fn=os.setsid)
        try:
            p.communicate()
        except (Exception, KeyboardInterrupt):
            print("tensorboard terminated")
            os.killpg(os.getpgid(p.pid), signal.SIGTERM)
            pass
 
    # cleanup when log_dir is temporary
    if is_tmp:
        shutil.rmtree(log_dir)
 
 

References print(), and read_constant_graph().

Referenced by _test().

write_constant_graph()

def tools.write_constant_graph	(		session,
			output_names,
			graph_path,
		**	kwargs
	)

Takes a TensorFlow *session* object (compatible with the v1 API), converts its contained graph
into a simpler version with variables translated into constant tensors, and saves it to a pb
file defined by *graph_path*. *output_numes* must be a list of names of output tensors to save.
In turn, TensorFlow internally determines which subgraph(s) to convert and save. All *kwargs*
are forwarded to :py:func:`tf.compat.v1.train.write_graph`. Intermediate output directories are
created, the output file is removed when already existing, and the absolute and normalized
output path is returned.

.. note::

    When used with TensorFlow v2, this function requires the v1 API compatibility layer. When
    :py:attr:`tf.compat.v1` is not available, a *NotImplementedError* is raised.

Definition at line 93 of file tools.py.

def write_constant_graph(session, output_names, graph_path, **kwargs):
    """
    Takes a TensorFlow *session* object (compatible with the v1 API), converts its contained graph
    into a simpler version with variables translated into constant tensors, and saves it to a pb
    file defined by *graph_path*. *output_numes* must be a list of names of output tensors to save.
    In turn, TensorFlow internally determines which subgraph(s) to convert and save. All *kwargs*
    are forwarded to :py:func:`tf.compat.v1.train.write_graph`. Intermediate output directories are
    created, the output file is removed when already existing, and the absolute and normalized
    output path is returned.
 
    .. note::
 
        When used with TensorFlow v2, this function requires the v1 API compatibility layer. When
        :py:attr:`tf.compat.v1` is not available, a *NotImplementedError* is raised.
    """
    # complain when the v1 compatibility layer is not existing
    if not tf1:
        raise NotImplementedError("the v1 compatibility layer of TensorFlow v2 (tf.compat.v1) is "
            "required by write_constant_graph, but missing")
 
    # convert the graph
    constant_graph = tf1.graph_util.convert_variables_to_constants(session,
        session.graph.as_graph_def(), output_names)
 
    # prepare the output path
    graph_path = os.path.normpath(os.path.abspath(graph_path))
    graph_dir, graph_name = os.path.split(graph_path)
    if not os.path.exists(graph_dir):
        os.makedirs(graph_dir)
    if os.path.exists(graph_path):
        os.remove(graph_path)
 
    # write the graph
    kwargs.setdefault("as_text", False)
    tf1.train.write_graph(constant_graph, graph_dir, graph_name, **kwargs)
 
    return graph_path
 
 

Referenced by _test().

Variable Documentation

__all__

tools.__all__

private

Definition at line 8 of file tools.py.

TF1

tools.TF1

Definition at line 23 of file tools.py.

Referenced by CalibrationAlgorithm.analyse(), PedsFullNoiseAlgorithm.analyse(), CalibrationScanAlgorithm.analyse(), EcalLaserAnalyzerYousi.analyze(), ResolutionCreator.beginJob(), HiCentralityBiasFilter.beginJob(), L1TRate_Offline.bookHistograms(), L1TRate.bookHistograms(), BSvsPVPlots(), BTagEntry.BTagEntry(), PPSAlignmentHarvester.buildModeGraph(), ApeEstimatorSummary.calculateApe(), PixelLumiDQM.calculateBunchMask(), SiPixelSCurveCalibrationAnalysis.calibrationSetup(), chi2toMinimize(), L1TOccupancyClient.compareWithStrip(), MuonBremsstrahlungSimulator.compute(), EcalTPGParamBuilder.computeLUT(), MuonTestSummary.doEnergyTests(), SideBandSubtract.doFastSubtraction(), PPSAlignmentHarvester.doMatch(), OuterTrackerMCHarvester.dqmEndJob(), SusyPostProcessor.dqmEndJob(), DTLocalTriggerSynchTest.dqmEndJob(), EcalZmassClient.dqmEndJob(), DTResolutionAnalysisTest.dqmEndJob(), Vx3DHLTAnalyzer.dqmEndLuminosityBlock(), DTResolutionTest.dqmEndLuminosityBlock(), Comparator.DrawGaussSigmaOverMeanXSlice(), Comparator.DrawGaussSigmaSlice(), Tutils.drawGFit(), DTNoiseComputation.endJob(), EcalLaserAnalyzerYousi.endJob(), analyzer::SiPixelLorentzAngle.endJob(), HOCalibAnalyzer.endJob(), L1TRate_Offline.endLuminosityBlock(), L1TRate.endLuminosityBlock(), ESRecHitFitAlgo.ESRecHitFitAlgo(), ESTimingTask.ESTimingTask(), DTMeanTimerFitter.evaluateVDriftAndReso(), LASPeakFinder.FindPeakIn(), BSFitter.Fit_d0phi(), Fit_MaximumPoint(), LA_Filler_Fitter.fit_width_profile(), BSFitter.Fit_z_chi2(), BeamMonitor.FitAndFill(), FakeBeamMonitor.FitAndFill(), SymmetryFit.fitfunction(), EgammaObjects.fitHistos(), CSCSimHitMatcher.fitHitsInChamber(), MuScleFitUtils.fitMass(), PPSAlignmentConfigESSource.fitProfile(), DTResidualFitter.fitResiduals(), SplitVertexResolution.fitResiduals(), TrackerOfflineValidationSummary.fitResiduals(), PVValHelper.fitResiduals(), TrackerOfflineValidation.fitResiduals(), SplitVertexResolution.fitResiduals_v0(), MuScleFitUtils.fitReso(), PFJetDQMPostProcessor.fitResponse(), TkLasBeamFitter.fitter(), DTTimeBoxFitter.fitTimeBox(), DTMeanTimerFitter.fitTMax(), fpeak_convoluted(), backgroundFunctionBase.functionForIntegral(), MuonTestSummary.GaussFit(), PFClient_JetRes.getHistogramParameters(), PFClient.getHistogramParameters(), PhiSymmetryCalibration.getKfactors(), CastorLedAnalysis.GetLedConst(), HcalLedAnalysis.GetLedConst(), BTagDifferentialPlot.getMistag(), GetMPV(), SiStripGainCosmicCalculator.getPeakOfLandau(), SiStripGainsPCLHarvester.getPeakOfLandau(), SiStripGainFromData.getPeakOfLandau(), SiStripGainFromCalibTree.getPeakOfLandau(), CastorPedestalAnalysis.GetPedConst(), HcalPedestalAnalysis.GetPedConst(), L1TRate.getXSexFitsOMDS(), L1TRate_Offline.getXSexFitsPython(), L1TRate.getXSexFitsPython(), reco::modules::HICaloCompatibleTrackSelector.HICaloCompatibleTrackSelector(), HiFJRhoFlowModulationProducer.HiFJRhoFlowModulationProducer(), MuonResidualsFitter.histogramChi2GaussianFit(), EvolutionECAL.InducedAbsorptionEM(), JetResolution.initialize(), LRHelpFunctions.initLRHistsAndFits(), fastsim::MuonBremsstrahlung.interact(), JetResolution.JetResolution(), L1TkMuCorrDynamicWindows.L1TkMuCorrDynamicWindows(), L1TkMuMantra.L1TkMuMantra(), CastorLedAnalysis.LedCastorHists(), HcalLedAnalysis.LedHBHEHists(), HcalLedAnalysis.LedHFHists(), HcalLedAnalysis.LedHOHists(), DQMGenericClient.limitedFit(), BTagCalibrationReader::BTagCalibrationReaderImpl.load(), LRHelpFunctions.LRHelpFunctions(), main(), TPedValues.makePlots(), EcalUncalibRecHitRecAnalFitAlgo< EBDataFrame >.makeRecHit(), GeometryComparisonPlotter.MakeTables(), multibsvspvplots(), MuScleFit.MuScleFit(), pat::ObjectResolutionCalc.ObjectResolutionCalc(), LA_Filler_Fitter.offset_slope(), JetResolution.parameter(), JetResolution.parameterEtaEval(), ParametrizedSubtractor.ParametrizedSubtractor(), MuonResidualsAngleFitter.plot(), MuonResidualsBfieldAngleFitter.plot(), MuonResiduals1DOFFitter.plot(), MuonResidualsPositionFitter.plot(), MuonResiduals6DOFrphiFitter.plot(), MuonResiduals5DOFFitter.plot(), MuonResiduals6DOFFitter.plot(), root.plot(), SymmetryFit.pol2_from_pol2(), TH2Analyzer.ProcessSlice(), PixelVertexProducerMedian.produce(), HIPixelMedianVtxProducer.produce(), CastorFastTowerProducer.produce(), PtHatRapReweightUserHook.PtHatRapReweightUserHook(), RapReweightUserHook.RapReweightUserHook(), LRHelpFunctions.readObsHistsAndFits(), LRHelpFunctions.recreateFitFct(), BackgroundHandler.rescale(), MuonTestSummary.ResidualCheck(), LA_Filler_Fitter.result(), DTLocalTriggerLutTest.runClientDiagnostic(), DTLocalTriggerSynchTest.runClientDiagnostic(), PVFitter.runFitter(), SamplingAlgorithm.SamplingAlgorithm(), MuMatchWindow.SetCentral(), MuMatchWindow.SetLower(), MuMatchWindow.SetUpper(), SiPixelGainCalibrationAnalysis.SiPixelGainCalibrationAnalysis(), smartGausProfile(), smearFunctionType7.smear(), LA_Filler_Fitter.summarize_ensembles(), root.tf1_t(), tmtt::Histos.trackerGeometryAnalysis(), CastorPedestalAnalysis.Trendings(), HcalPedestalAnalysis.Trendings(), UnbinnedFitDeDxEstimator.UnbinnedFitDeDxEstimator(), TEcnaHistos.ViewEBGrid(), TEcnaHistos.ViewSCGrid(), TEcnaHistos.ViewSMGrid(), TEcnaHistos.ViewTowerGrid(), sistrip::MeasureLA.write_report_text_ms(), PPSAlignmentHarvester.xAlignmentRelative(), L1TOccupancyClient.xySymmetry(), and PPSAlignmentHarvester.yAlignment().

tf1

tools.tf1

Definition at line 31 of file tools.py.

TF2

tools.TF2

Definition at line 24 of file tools.py.

Referenced by TtFullLepKinSolver.TtFullLepKinSolver().