condformats_serialization_generate Namespace Reference

Classes
class	SerializationCodeGenerator

Functions
def	get_basic_type_string (node)
def	get_clang_version ()
def	get_default_gcc_search_paths (gcc='g++', language='c++')
def	get_diagnostics (translation_unit)
def	get_flags (product_name, flags)
def	get_serializable_classes_members (node, all_template_types=None, namespace='', only_from_path=None)
def	get_statement (node)
def	get_type_string (node)
def	is_definition_by_loc (node)
def	is_friend_decl (memkind)
def	is_serializable_class (node)
def	is_serializable_class_manual (node)
def	log_flags (name, flags)
def	main ()
def	sanitise (var)
def	split_path (path)

Variables
string	__author__ = 'Miguel Ojeda'
string	__copyright__ = 'Copyright 2014, CERN'
list	__credits__ = ['Giacomo Govi', 'Miguel Ojeda', 'Andreas Pfeiffer']
string	__email__ = 'mojedasa@cern.ch'
string	__license__ = 'Unknown'
string	__maintainer__ = 'Miguel Ojeda'
	clang_version = None
string	headers_template
string	instantiation_template
string	serialize_method_base_object_template = ' ar & boost::serialization::make_nvp("{base_object_name_sanitised}", boost::serialization::base_object<{base_object_name}>(*this));'
string	serialize_method_begin_template
string	serialize_method_end
string	serialize_method_member_template = ''' ar & boost::serialization::make_nvp("{member_name_sanitised}", {member_name});'''
	skip_namespaces

Function Documentation

get_basic_type_string()

def condformats_serialization_generate.get_basic_type_string

(

node

)

Definition at line 122 of file condformats_serialization_generate.py.

Referenced by get_serializable_classes_members(), and get_type_string().

def get_basic_type_string(node):
    typekinds = {
        clang.cindex.TypeKind.BOOL: 'bool',
        clang.cindex.TypeKind.INT: 'int',
        clang.cindex.TypeKind.LONG: 'long',
        clang.cindex.TypeKind.UINT: 'unsigned int',
        clang.cindex.TypeKind.ULONG: 'unsigned long',
        clang.cindex.TypeKind.FLOAT: 'float',
        clang.cindex.TypeKind.DOUBLE: 'double',
    }

    if node.type.kind not in typekinds:
        raise Exception('Not a known basic type.')

    return typekinds[node.type.kind]

get_clang_version()

def condformats_serialization_generate.get_clang_version

(

)

Extract clang version and set global clang_version and also return the same value.

Definition at line 348 of file condformats_serialization_generate.py.

References edm.decode(), createfilelist.int, submitPVValidationJobs.split(), and nano_mu_digi_cff.strip.

Referenced by is_friend_decl().

def get_clang_version():
    """Extract clang version and set global clang_version and also return the same value."""
    global clang_version
    if clang_version is not None:
        return clang_version
    command = "clang --version | grep 'clang version' | sed 's/clang version//'"
    logging.debug("Running: {0}".format(command))
    (clang_version_major, clang_version_minor, clang_version_patchlevel) = subprocess.check_output(command, shell=True).splitlines()[0].decode('ascii').strip().split(" ")[0].split('.', 3)
    clang_version = (int(clang_version_major), int(clang_version_minor), int(clang_version_patchlevel))
    logging.debug("Detected Clang version: {0}".format(clang_version))
    return clang_version

get_default_gcc_search_paths()

def condformats_serialization_generate.get_default_gcc_search_paths	(		gcc = 'g++',
			language = 'c++'
	)

Definition at line 386 of file condformats_serialization_generate.py.

def get_default_gcc_search_paths(gcc = 'g++', language = 'c++'):
    command = 'echo "" | %s -x%s -v -E - 2>&1' % (gcc, language)
    logging.debug('Running: %s', command)

    paths = []
    in_list = False
    for line in [l.decode("ascii") for l in subprocess.check_output(command, shell=True).splitlines()]:
        if in_list:
            if line == 'End of search list.':
                break

            path = os.path.normpath(line.strip())

            # Intrinsics not handled by clang
            # Note that /lib/gcc is found in other paths if not normalized,
            # so has to go after normpath()
            if '/lib/gcc/' in path:
                continue

            paths.append('-I%s' % path)

        else:
            if line == '#include <...> search starts here:':
                in_list = True

    if not in_list:
        raise Exception('Default GCC search paths not found.')

    return paths

get_diagnostics()

def condformats_serialization_generate.get_diagnostics

(

translation_unit

)

Definition at line 376 of file condformats_serialization_generate.py.

References genParticles_cff.map.

def get_diagnostics(translation_unit):
    return map(lambda diag: {
        'severity' : diag.severity,
        'location' : diag.location,
        'spelling' : diag.spelling,
        'ranges' : diag.ranges,
        'fixits' : diag.fixits,
    }, translation_unit.diagnostics)

get_flags()

def condformats_serialization_generate.get_flags	(		product_name,
			flags
	)

Definition at line 343 of file condformats_serialization_generate.py.

def get_flags(product_name, flags):
    command = "scram b echo_%s_%s | tail -1 | cut -d '=' -f '2-' | xargs -n1" % (product_name, flags)
    logging.debug('Running: %s', command)
    return subprocess.check_output(command, shell=True).splitlines()

get_serializable_classes_members()

def condformats_serialization_generate.get_serializable_classes_members	(		node,
			all_template_types = None,
			namespace = '',
			only_from_path = None
	)

Definition at line 147 of file condformats_serialization_generate.py.

References get_basic_type_string(), get_statement(), get_type_string(), is_definition_by_loc(), is_friend_decl(), is_serializable_class(), is_serializable_class_manual(), and join().

def get_serializable_classes_members(node, all_template_types=None, namespace='', only_from_path=None):
    if all_template_types is None:
        all_template_types = []

    logging.debug('%s', (node.spelling, all_template_types, namespace))
    results = {}
    for child in node.get_children():
        if child.kind == clang.cindex.CursorKind.NAMESPACE:
            # If we are in the root namespace, let's skip some common, big
            # namespaces to improve speed and avoid serializing those.
            if namespace == '':
                if child.spelling in skip_namespaces:
                    continue

                # This skips compiler-specific stuff as well (e.g. __gnucxx...)
                if child.spelling.startswith('_'):
                    continue

            logging.debug('Going into namespace %s', child.spelling)

            results.update(get_serializable_classes_members(child, all_template_types, namespace + child.spelling + '::', only_from_path))
            continue

        if child.kind in [clang.cindex.CursorKind.CLASS_DECL, clang.cindex.CursorKind.STRUCT_DECL, clang.cindex.CursorKind.CLASS_TEMPLATE] and is_definition_by_loc(child):
            logging.debug('Found struct/class/template definition: %s', child.spelling if child.spelling else '<anonymous>')

            if only_from_path is not None \
                and child.location.file is not None \
                and not child.location.file.name.startswith(only_from_path):
                logging.debug('Skipping since it is an external of this package: %s', child.spelling)
                continue

            serializable = is_serializable_class(child)
            if serializable:
                if child.spelling == '':
                    raise Exception('It is not possible to serialize anonymous/unnamed structs/classes.')

                if is_serializable_class_manual(child):
                    logging.info('Found manual serializable struct/class/template: %s', child.spelling)
                    continue

                logging.info('Found serializable struct/class/template: %s', child.spelling)

            template_types = []
            base_objects = []
            members = []
            transients = []
            after_serialize = False
            after_serialize_count = 0
            for member in child.get_children():
                if after_serialize:
                    if after_serialize_count == 2:
                        after_serialize = False
                    else:
                        after_serialize_count = after_serialize_count + 1

                        if not is_friend_decl(member.kind):
                            raise Exception('Expected unexposed declaration (friend) after serialize() but found something else: looks like the COND_SERIALIZABLE macro has been changed without updating the script.')

                        if 'COND_SERIALIZABLE' not in get_statement(member):
                            raise Exception('Could not find COND_SERIALIZABLE in the statement of the expected unexposed declarations (friends) after serialize(). Please fix the script/macro.')

                        logging.debug('Skipping expected unexposed declaration (friend) after serialize().')
                        continue

                # Template type parameters (e.g. <typename T>)
                if member.kind == clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER:
                    logging.info('    Found template type parameter: %s', member.spelling)
                    template_types.append(('typename', member.spelling))

                # Template non-type parameters (e.g. <int N>)
                elif member.kind == clang.cindex.CursorKind.TEMPLATE_NON_TYPE_PARAMETER:
                    type_string = get_type_string(member)
                    if not type_string: 
                        type_string = get_basic_type_string(member)
                    logging.info('    Found template non-type parameter: %s %s', type_string, member.spelling)
                    template_types.append((type_string, member.spelling))

                # Base objects
                elif member.kind == clang.cindex.CursorKind.CXX_BASE_SPECIFIER:
                    # FIXME: .displayname gives sometimes things like "class mybase"
                    base_object = member.displayname
                    prefix = 'class '
                    if base_object.startswith(prefix):
                        base_object = base_object[len(prefix):]
                    logging.info('    Found base object: %s', base_object)
                    base_objects.append(base_object)

                # Member variables
                elif member.kind == clang.cindex.CursorKind.FIELD_DECL and is_definition_by_loc(member):
                    # While clang 3.3 does not ignore unrecognized attributes
                    # (see http://llvm.org/viewvc/llvm-project?revision=165082&view=revision )
                    # for some reason they do not appear in the bindings yet
                    # so we just do it ourselves.

                    # FIXME: To simplify and avoid parsing C++ ourselves, our transient
                    # attribute applies to *all* the variables declared in the same statement.
                    if 'COND_TRANSIENT' not in get_statement(member):
                        logging.info('    Found member variable: %s', member.spelling)
                        members.append(member.spelling)
                    else:
                        if serializable:
                            logging.info('    Found transient member variable: %s', member.spelling)
                            transients.append(member.spelling)
                        else:
                            raise Exception('Transient %s found for non-serializable class %s', member.spelling, child.spelling)

                elif member.kind == clang.cindex.CursorKind.FUNCTION_TEMPLATE and member.spelling == 'serialize':
                    after_serialize = True
                    logging.debug('Found serialize() method, skipping next two children which must be unexposed declarations.')

                elif member.kind in frozenset([
                    # For safety, we list all known kinds that we need to skip
                    # and raise in unknown cases (this helps catching problems
                    # with undefined classes)
                    clang.cindex.CursorKind.CONSTRUCTOR,
                    clang.cindex.CursorKind.DESTRUCTOR,
                    clang.cindex.CursorKind.CXX_METHOD,
                    clang.cindex.CursorKind.CXX_ACCESS_SPEC_DECL,
                    clang.cindex.CursorKind.FUNCTION_TEMPLATE,
                    clang.cindex.CursorKind.TYPEDEF_DECL,
                    clang.cindex.CursorKind.CLASS_DECL,
                    clang.cindex.CursorKind.ENUM_DECL,
                    clang.cindex.CursorKind.VAR_DECL,
                    clang.cindex.CursorKind.STRUCT_DECL,
                    clang.cindex.CursorKind.UNION_DECL,
                    clang.cindex.CursorKind.CONVERSION_FUNCTION,
                    clang.cindex.CursorKind.TYPE_REF,
                    clang.cindex.CursorKind.DECL_REF_EXPR,
                    clang.cindex.CursorKind.CLASS_TEMPLATE,
                    clang.cindex.CursorKind.TYPE_ALIAS_DECL,
                ]):
                    logging.debug('Skipping member: %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind)

                elif is_friend_decl(member.kind):
                    statement = get_statement(member)

                    # Friends are unexposed but they are not data to serialize
                    if 'friend' in statement:
                        # If we know about them, skip the warning
                        if \
                            'friend class ' in statement or \
                            'friend struct ' in statement or \
                            'friend std::ostream& operator<<(' in statement or \
                            'friend std::istream& operator>>(' in statement:
                            logging.debug('Skipping known friend: %s', statement.splitlines()[0])
                            continue

                        # Otherwise warn
                        logging.warning('Unexposed declaration that looks like a friend declaration -- please check: %s %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind, statement)
                        continue

                    raise Exception('Unexposed declaration. This probably means (at the time of writing) that an unknown class was found (may happen, for instance, when the compiler does not find the headers for std::vector, i.e. missing -I option): %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))

                else:
                    statement = get_statement(member)
                    raise Exception('Unknown kind. Please fix the script: %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))

            if template_types:
                template_use = '%s<%s>' % (child.spelling, ', '.join([template_type_name for (_, template_type_name) in template_types]))
            else:
                template_use = child.spelling

            new_namespace = namespace + template_use

            new_all_template_types = all_template_types + [template_types]

            results[new_namespace] = (child, serializable, new_all_template_types, base_objects, members, transients)

            results.update(get_serializable_classes_members(child, new_all_template_types, new_namespace + '::', only_from_path))

    for (klass, (node, serializable, all_template_types, base_objects, members, transients)) in results.items():
        if serializable and len(members) == 0:
            logging.info('No non-transient members found for serializable class %s', klass)

    return results

get_statement()

def condformats_serialization_generate.get_statement

(

node

)

Definition at line 105 of file condformats_serialization_generate.py.

References edm.decode().

Referenced by get_serializable_classes_members().

def get_statement(node):
    # For some cursor kinds, their location is empty (e.g. translation units
    # and attributes); either because of a bug or because they do not have
    # a meaningful 'start' -- however, the extent is always available
    if node.extent.start.file is None:
        return None

    filename = node.extent.start.file.name
    start = node.extent.start.offset
    end = node.extent.end.offset

    with open(filename, 'rb') as fd:
        source = fd.read().decode('latin-1')

    return source[start:source.find(';', end)]

get_type_string()

def condformats_serialization_generate.get_type_string

(

node

)

Definition at line 139 of file condformats_serialization_generate.py.

References get_basic_type_string().

Referenced by get_serializable_classes_members().

def get_type_string(node):
    spelling = node.type.get_declaration().spelling
    if spelling is not None:
        return spelling

    return get_basic_type_string(node)

is_definition_by_loc()

def condformats_serialization_generate.is_definition_by_loc

(

node

)

Definition at line 73 of file condformats_serialization_generate.py.

Referenced by get_serializable_classes_members(), is_serializable_class(), and is_serializable_class_manual().

def is_definition_by_loc(node):
    if node.get_definition() is None:
        return False
    if node.location is None or node.get_definition().location is None:
        return False
    return node.location == node.get_definition().location

is_friend_decl()

def condformats_serialization_generate.is_friend_decl

(

memkind

)

Check if declaration is a friend

Definition at line 360 of file condformats_serialization_generate.py.

References get_clang_version().

Referenced by get_serializable_classes_members().

def is_friend_decl(memkind):
    """Check if declaration is a friend"""
    clangv = get_clang_version()
    if clangv >= (4, 0, 0):
        return memkind == clang.cindex.CursorKind.FRIEND_DECL
    else:
        return memkind == clang.cindex.CursorKind.UNEXPOSED_DECL
    return false

is_serializable_class()

def condformats_serialization_generate.is_serializable_class

(

node

)

Definition at line 80 of file condformats_serialization_generate.py.

References is_definition_by_loc().

Referenced by get_serializable_classes_members().

def is_serializable_class(node):
    for child in node.get_children():
        if child.spelling != 'serialize' or child.kind != clang.cindex.CursorKind.FUNCTION_TEMPLATE or is_definition_by_loc(child):
            continue

        if [(x.spelling, x.kind, is_definition_by_loc(x), x.type.kind) for x in child.get_children()] != [
            ('Archive', clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER, True, clang.cindex.TypeKind.UNEXPOSED),
            ('ar', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.LVALUEREFERENCE),
            ('version', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.UINT),
        ]:
            continue

        return True

    return False

is_serializable_class_manual()

def condformats_serialization_generate.is_serializable_class_manual

(

node

)

Definition at line 97 of file condformats_serialization_generate.py.

References is_definition_by_loc().

Referenced by get_serializable_classes_members().

def is_serializable_class_manual(node):
    for child in node.get_children():
        if child.spelling == 'cond_serialization_manual' and child.kind == clang.cindex.CursorKind.CXX_METHOD and not is_definition_by_loc(child):
            return True

    return False

log_flags()

def condformats_serialization_generate.log_flags	(		name,
			flags
	)

Definition at line 369 of file condformats_serialization_generate.py.

def log_flags(name, flags):
    logging.debug('%s = [', name)
    for flag in flags:
        logging.debug('    %s', flag)
    logging.debug(']')

main()

def condformats_serialization_generate.main

(

)

Definition at line 571 of file condformats_serialization_generate.py.

References generate().

def main():
    parser = argparse.ArgumentParser(description='CMS Condition DB Serialization generator.')
    parser.add_argument('--verbose', '-v', action='count', help='Verbosity level. -v reports debugging information.', default=0)
    parser.add_argument('--output' , '-o', action='store', help='Specifies the path to the output file written. Default: src/Serialization.cc')
    parser.add_argument('--package', '-p', action='store', help='Specifies the path to the package to be processed. Default: the actual package')

    opts, args = parser.parse_known_args()

    logLevel = logging.INFO
    if opts.verbose < 1 and opts.output and opts.package:   # assume we're called by scram and reduce logging - but only if no verbose is requested
        logLevel = logging.WARNING

    if opts.verbose >= 1: 
        logLevel = logging.DEBUG

    logging.basicConfig(
        format = '[%(asctime)s] %(levelname)s: %(message)s',
        level = logLevel,
    )

    if opts.package:  # we got a directory name to process, assume it's from scram and remove the last ('/src') dir from the path
        pkgDir = opts.package
        if pkgDir.endswith('/src') :
            pkgDir, srcDir = os.path.split( opts.package )
        os.chdir( pkgDir )
        logging.info("Processing package in %s " % pkgDir)

    if opts.output:
        logging.info("Writing serialization code to %s " % opts.output)

    SerializationCodeGenerator( scramFlags=args[1:] ).generate( opts.output )

sanitise()

def condformats_serialization_generate.sanitise

(

var

)

Definition at line 416 of file condformats_serialization_generate.py.

Referenced by condformats_serialization_generate.SerializationCodeGenerator.generate().

def sanitise(var):
    return re.sub('[^a-zA-Z0-9.,-:]', '-', var)

split_path()

def condformats_serialization_generate.split_path

(

path

)

Definition at line 325 of file condformats_serialization_generate.py.

def split_path(path):
    folders = []

    while True:
        path, folder = os.path.split(path)

        if folder != '':
            folders.append(folder)
        else:
            if path != '':
                folders.append(path)
            break

    folders.reverse()

    return folders

Variable Documentation

__author__

string condformats_serialization_generate.__author__ = 'Miguel Ojeda'

private

Definition at line 17 of file condformats_serialization_generate.py.

__copyright__

string condformats_serialization_generate.__copyright__ = 'Copyright 2014, CERN'

private

Definition at line 18 of file condformats_serialization_generate.py.

__credits__

list condformats_serialization_generate.__credits__ = ['Giacomo Govi', 'Miguel Ojeda', 'Andreas Pfeiffer']

private

Definition at line 19 of file condformats_serialization_generate.py.

__email__

string condformats_serialization_generate.__email__ = 'mojedasa@cern.ch'

private

Definition at line 22 of file condformats_serialization_generate.py.

__license__

string condformats_serialization_generate.__license__ = 'Unknown'

private

Definition at line 20 of file condformats_serialization_generate.py.

__maintainer__

string condformats_serialization_generate.__maintainer__ = 'Miguel Ojeda'

private

Definition at line 21 of file condformats_serialization_generate.py.

clang_version

condformats_serialization_generate.clang_version = None

Definition at line 33 of file condformats_serialization_generate.py.

headers_template

string condformats_serialization_generate.headers_template

Initial value:

=  '''
#include "{headers}"

#include <boost/serialization/base_object.hpp>
#include <boost/serialization/nvp.hpp>
#include <boost/serialization/export.hpp>

#include "CondFormats/Serialization/interface/Equal.h"
#include "CondFormats/Serialization/interface/Instantiate.h"

'''

Definition at line 35 of file condformats_serialization_generate.py.

instantiation_template

string condformats_serialization_generate.instantiation_template

Initial value:

=  '''COND_SERIALIZATION_INSTANTIATE({klass});
'''

Definition at line 58 of file condformats_serialization_generate.py.

serialize_method_base_object_template

string condformats_serialization_generate.serialize_method_base_object_template = ' ar & boost::serialization::make_nvp("{base_object_name_sanitised}", boost::serialization::base_object<{base_object_name}>(*this));'

Definition at line 51 of file condformats_serialization_generate.py.

serialize_method_begin_template

string condformats_serialization_generate.serialize_method_begin_template

Initial value:

=  '''template <class Archive>
void {klass}::serialize(Archive & ar, const unsigned int)
{{'''

Definition at line 47 of file condformats_serialization_generate.py.

serialize_method_end

string condformats_serialization_generate.serialize_method_end

Initial value:

=  '''}
'''

Definition at line 55 of file condformats_serialization_generate.py.

serialize_method_member_template

string condformats_serialization_generate.serialize_method_member_template = ''' ar & boost::serialization::make_nvp("{member_name_sanitised}", {member_name});'''

Definition at line 53 of file condformats_serialization_generate.py.

skip_namespaces

condformats_serialization_generate.skip_namespaces

Initial value:

=  frozenset([
    # Do not go inside anonymous namespaces (static)
    '',

    # Do not go inside some standard namespaces
    'std', 'boost', 'mpl_', 'boost_swap_impl',

    # Do not go inside some big namespaces coming from externals
    'ROOT', 'edm', 'ora', 'coral', 'CLHEP', 'Geom', 'HepGeom',
])

Definition at line 62 of file condformats_serialization_generate.py.