condformats_serialization_generate Namespace Reference

Classes
class	SerializationCodeGenerator

Functions
def	get_basic_type_string (node)
def	get_clang_version ()
def	get_default_gcc_search_paths (gcc='g++', language='c++')
def	get_diagnostics (translation_unit)
def	get_flags (product_name, flags)
def	get_serializable_classes_members (node, all_template_types=None, namespace='', only_from_path=None)
def	get_statement (node)
def	get_type_string (node)
def	is_definition_by_loc (node)
def	is_friend_decl (memkind)
def	is_serializable_class (node)
def	is_serializable_class_manual (node)
def	log_flags (name, flags)
def	main ()
def	sanitise (var)
def	split_path (path)

Detailed Description

CMS Conditions DB Serialization generator.

Generates the non-intrusive serialization code required for the classes
marked with the COND_SERIALIZABLE macro.

The code was taken from the prototype that did many other things as well
(finding transients, marking serializable classes, etc.). After removing
everything but what is required to build the serialization, the code was
made more robust and cleaned up a bit to be integrated on the BoostIO IB.
However, the code still needs to be restructured a bit more to improve
readability (e.g. name some constants, use a template engine, ask for
clang's bindings to be installed along clang itself, etc.).

Function Documentation

def condformats_serialization_generate.get_basic_type_string

(

node

)

Definition at line 121 of file condformats_serialization_generate.py.

Referenced by get_serializable_classes_members(), and get_type_string().

def get_basic_type_string(node):
    typekinds = {
        clang.cindex.TypeKind.BOOL: 'bool',
        clang.cindex.TypeKind.INT: 'int',
        clang.cindex.TypeKind.LONG: 'long',
        clang.cindex.TypeKind.UINT: 'unsigned int',
        clang.cindex.TypeKind.ULONG: 'unsigned long',
        clang.cindex.TypeKind.FLOAT: 'float',
        clang.cindex.TypeKind.DOUBLE: 'double',
    }

    if node.type.kind not in typekinds:
        raise Exception('Not a known basic type.')

    return typekinds[node.type.kind]

def condformats_serialization_generate.get_clang_version

(

)

Extract clang version and set global clang_version and also return the same value.

Definition at line 347 of file condformats_serialization_generate.py.

References createfilelist.int, split, and digitizers_cfi.strip.

Referenced by is_friend_decl().

def get_clang_version():
    """Extract clang version and set global clang_version and also return the same value."""
    global clang_version
    if clang_version is not None:
        return clang_version
    command = "clang --version | grep 'clang version' | sed 's/clang version//'"
    logging.debug("Running: {0}".format(command))
    (clang_version_major, clang_version_minor, clang_version_patchlevel) = subprocess.check_output(command, shell=True).splitlines()[0].strip().split('.', 3)
    clang_version = (int(clang_version_major), int(clang_version_minor), int(clang_version_patchlevel))
    logging.debug("Detected Clang version: {0}".format(clang_version))
    return clang_version

def condformats_serialization_generate.get_default_gcc_search_paths	(		gcc = 'g++',
			language = 'c++'
	)

Definition at line 385 of file condformats_serialization_generate.py.

def get_default_gcc_search_paths(gcc = 'g++', language = 'c++'):
    command = 'echo "" | %s -x%s -v -E - 2>&1' % (gcc, language)
    logging.debug('Running: %s', command)

    paths = []
    in_list = False
    for line in subprocess.check_output(command, shell=True).splitlines():
        if in_list:
            if line == 'End of search list.':
                break

            path = os.path.normpath(line.strip())

            # Intrinsics not handled by clang
            # Note that /lib/gcc is found in other paths if not normalized,
            # so has to go after normpath()
            if '/lib/gcc/' in path:
                continue

            paths.append('-I%s' % path)

        else:
            if line == '#include <...> search starts here:':
                in_list = True

    if not in_list:
        raise Exception('Default GCC search paths not found.')

    return paths

def condformats_serialization_generate.get_diagnostics

(

translation_unit

)

Definition at line 375 of file condformats_serialization_generate.py.

References genParticles_cff.map.

def get_diagnostics(translation_unit):
    return map(lambda diag: {
        'severity' : diag.severity,
        'location' : diag.location,
        'spelling' : diag.spelling,
        'ranges' : diag.ranges,
        'fixits' : diag.fixits,
    }, translation_unit.diagnostics)

def condformats_serialization_generate.get_flags	(		product_name,
			flags
	)

Definition at line 342 of file condformats_serialization_generate.py.

def get_flags(product_name, flags):
    command = "scram b echo_%s_%s | tail -1 | cut -d '=' -f '2-' | xargs -n1" % (product_name, flags)
    logging.debug('Running: %s', command)
    return subprocess.check_output(command, shell=True).splitlines()

def condformats_serialization_generate.get_serializable_classes_members	(		node,
			all_template_types = None,
			namespace = '',
			only_from_path = None
	)

Definition at line 146 of file condformats_serialization_generate.py.

References get_basic_type_string(), get_statement(), get_type_string(), is_definition_by_loc(), is_friend_decl(), is_serializable_class(), is_serializable_class_manual(), and join().

def get_serializable_classes_members(node, all_template_types=None, namespace='', only_from_path=None):
    if all_template_types is None:
        all_template_types = []

    logging.debug('%s', (node.spelling, all_template_types, namespace))
    results = {}
    for child in node.get_children():
        if child.kind == clang.cindex.CursorKind.NAMESPACE:
            # If we are in the root namespace, let's skip some common, big
            # namespaces to improve speed and avoid serializing those.
            if namespace == '':
                if child.spelling in skip_namespaces:
                    continue

                # This skips compiler-specific stuff as well (e.g. __gnucxx...)
                if child.spelling.startswith('_'):
                    continue

            logging.debug('Going into namespace %s', child.spelling)

            results.update(get_serializable_classes_members(child, all_template_types, namespace + child.spelling + '::', only_from_path))
            continue

        if child.kind in [clang.cindex.CursorKind.CLASS_DECL, clang.cindex.CursorKind.STRUCT_DECL, clang.cindex.CursorKind.CLASS_TEMPLATE] and is_definition_by_loc(child):
            logging.debug('Found struct/class/template definition: %s', child.spelling if child.spelling else '<anonymous>')

            if only_from_path is not None \
                and child.location.file is not None \
                and not child.location.file.name.startswith(only_from_path):
                logging.debug('Skipping since it is an external of this package: %s', child.spelling)
                continue

            serializable = is_serializable_class(child)
            if serializable:
                if child.spelling == '':
                    raise Exception('It is not possible to serialize anonymous/unnamed structs/classes.')

                if is_serializable_class_manual(child):
                    logging.info('Found manual serializable struct/class/template: %s', child.spelling)
                    continue

                logging.info('Found serializable struct/class/template: %s', child.spelling)

            template_types = []
            base_objects = []
            members = []
            transients = []
            after_serialize = False
            after_serialize_count = 0
            for member in child.get_children():
                if after_serialize:
                    if after_serialize_count == 2:
                        after_serialize = False
                    else:
                        after_serialize_count = after_serialize_count + 1

                        if not is_friend_decl(member.kind):
                            raise Exception('Expected unexposed declaration (friend) after serialize() but found something else: looks like the COND_SERIALIZABLE macro has been changed without updating the script.')

                        if 'COND_SERIALIZABLE' not in get_statement(member):
                            raise Exception('Could not find COND_SERIALIZABLE in the statement of the expected unexposed declarations (friends) after serialize(). Please fix the script/macro.')

                        logging.debug('Skipping expected unexposed declaration (friend) after serialize().')
                        continue

                # Template type parameters (e.g. <typename T>)
                if member.kind == clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER:
                    logging.info('    Found template type parameter: %s', member.spelling)
                    template_types.append(('typename', member.spelling))

                # Template non-type parameters (e.g. <int N>)
                elif member.kind == clang.cindex.CursorKind.TEMPLATE_NON_TYPE_PARAMETER:
                    type_string = get_type_string(member)
                    if not type_string: 
                        type_string = get_basic_type_string(member)
                    logging.info('    Found template non-type parameter: %s %s', type_string, member.spelling)
                    template_types.append((type_string, member.spelling))

                # Base objects
                elif member.kind == clang.cindex.CursorKind.CXX_BASE_SPECIFIER:
                    # FIXME: .displayname gives sometimes things like "class mybase"
                    base_object = member.displayname
                    prefix = 'class '
                    if base_object.startswith(prefix):
                        base_object = base_object[len(prefix):]
                    logging.info('    Found base object: %s', base_object)
                    base_objects.append(base_object)

                # Member variables
                elif member.kind == clang.cindex.CursorKind.FIELD_DECL and is_definition_by_loc(member):
                    # While clang 3.3 does not ignore unrecognized attributes
                    # (see http://llvm.org/viewvc/llvm-project?revision=165082&view=revision )
                    # for some reason they do not appear in the bindings yet
                    # so we just do it ourselves.

                    # FIXME: To simplify and avoid parsing C++ ourselves, our transient
                    # attribute applies to *all* the variables declared in the same statement.
                    if 'COND_TRANSIENT' not in get_statement(member):
                        logging.info('    Found member variable: %s', member.spelling)
                        members.append(member.spelling)
                    else:
                        if serializable:
                            logging.info('    Found transient member variable: %s', member.spelling)
                            transients.append(member.spelling)
                        else:
                            raise Exception('Transient %s found for non-serializable class %s', member.spelling, child.spelling)

                elif member.kind == clang.cindex.CursorKind.FUNCTION_TEMPLATE and member.spelling == 'serialize':
                    after_serialize = True
                    logging.debug('Found serialize() method, skipping next two children which must be unexposed declarations.')

                elif member.kind in frozenset([
                    # For safety, we list all known kinds that we need to skip
                    # and raise in unknown cases (this helps catching problems
                    # with undefined classes)
                    clang.cindex.CursorKind.CONSTRUCTOR,
                    clang.cindex.CursorKind.DESTRUCTOR,
                    clang.cindex.CursorKind.CXX_METHOD,
                    clang.cindex.CursorKind.CXX_ACCESS_SPEC_DECL,
                    clang.cindex.CursorKind.FUNCTION_TEMPLATE,
                    clang.cindex.CursorKind.TYPEDEF_DECL,
                    clang.cindex.CursorKind.CLASS_DECL,
                    clang.cindex.CursorKind.ENUM_DECL,
                    clang.cindex.CursorKind.VAR_DECL,
                    clang.cindex.CursorKind.STRUCT_DECL,
                    clang.cindex.CursorKind.UNION_DECL,
                    clang.cindex.CursorKind.CONVERSION_FUNCTION,
                    clang.cindex.CursorKind.TYPE_REF,
                    clang.cindex.CursorKind.DECL_REF_EXPR,
                    clang.cindex.CursorKind.CLASS_TEMPLATE,
                    clang.cindex.CursorKind.TYPE_ALIAS_DECL,
                ]):
                    logging.debug('Skipping member: %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind)

                elif is_friend_decl(member.kind):
                    statement = get_statement(member)

                    # Friends are unexposed but they are not data to serialize
                    if 'friend' in statement:
                        # If we know about them, skip the warning
                        if \
                            'friend class ' in statement or \
                            'friend struct ' in statement or \
                            'friend std::ostream& operator<<(' in statement or \
                            'friend std::istream& operator>>(' in statement:
                            logging.debug('Skipping known friend: %s', statement.splitlines()[0])
                            continue

                        # Otherwise warn
                        logging.warning('Unexposed declaration that looks like a friend declaration -- please check: %s %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind, statement)
                        continue

                    raise Exception('Unexposed declaration. This probably means (at the time of writing) that an unknown class was found (may happen, for instance, when the compiler does not find the headers for std::vector, i.e. missing -I option): %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))

                else:
                    statement = get_statement(member)
                    raise Exception('Unknown kind. Please fix the script: %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))

            if template_types:
                template_use = '%s<%s>' % (child.spelling, ', '.join([template_type_name for (_, template_type_name) in template_types]))
            else:
                template_use = child.spelling

            new_namespace = namespace + template_use

            new_all_template_types = all_template_types + [template_types]

            results[new_namespace] = (child, serializable, new_all_template_types, base_objects, members, transients)

            results.update(get_serializable_classes_members(child, new_all_template_types, new_namespace + '::', only_from_path))

    for (klass, (node, serializable, all_template_types, base_objects, members, transients)) in results.items():
        if serializable and len(members) == 0:
            logging.info('No non-transient members found for serializable class %s', klass)

    return results

def condformats_serialization_generate.get_statement

(

node

)

Definition at line 104 of file condformats_serialization_generate.py.

Referenced by get_serializable_classes_members().

def get_statement(node):
    # For some cursor kinds, their location is empty (e.g. translation units
    # and attributes); either because of a bug or because they do not have
    # a meaningful 'start' -- however, the extent is always available
    if node.extent.start.file is None:
        return None

    filename = node.extent.start.file.name
    start = node.extent.start.offset
    end = node.extent.end.offset

    with open(filename, 'rb') as fd:
        source = fd.read()

    return source[start:source.find(';', end)]

def condformats_serialization_generate.get_type_string

(

node

)

Definition at line 138 of file condformats_serialization_generate.py.

References get_basic_type_string().

Referenced by get_serializable_classes_members().

def get_type_string(node):
    spelling = node.type.get_declaration().spelling
    if spelling is not None:
        return spelling

    return get_basic_type_string(node)

def condformats_serialization_generate.is_definition_by_loc

(

node

)

Definition at line 72 of file condformats_serialization_generate.py.

Referenced by get_serializable_classes_members(), is_serializable_class(), and is_serializable_class_manual().

def is_definition_by_loc(node):
    if node.get_definition() is None:
        return False
    if node.location is None or node.get_definition().location is None:
        return False
    return node.location == node.get_definition().location

def condformats_serialization_generate.is_friend_decl

(

memkind

)

Check if declaration is a friend

Definition at line 359 of file condformats_serialization_generate.py.

References get_clang_version().

Referenced by get_serializable_classes_members().

def is_friend_decl(memkind):
    """Check if declaration is a friend"""
    clangv = get_clang_version()
    if clangv >= (4, 0, 0):
        return memkind == clang.cindex.CursorKind.FRIEND_DECL
    else:
        return memkind == clang.cindex.CursorKind.UNEXPOSED_DECL
    return false

def condformats_serialization_generate.is_serializable_class

(

node

)

Definition at line 79 of file condformats_serialization_generate.py.

References is_definition_by_loc().

Referenced by get_serializable_classes_members().

def is_serializable_class(node):
    for child in node.get_children():
        if child.spelling != 'serialize' or child.kind != clang.cindex.CursorKind.FUNCTION_TEMPLATE or is_definition_by_loc(child):
            continue

        if [(x.spelling, x.kind, is_definition_by_loc(x), x.type.kind) for x in child.get_children()] != [
            ('Archive', clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER, True, clang.cindex.TypeKind.UNEXPOSED),
            ('ar', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.LVALUEREFERENCE),
            ('version', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.UINT),
        ]:
            continue

        return True

    return False

def condformats_serialization_generate.is_serializable_class_manual

(

node

)

Definition at line 96 of file condformats_serialization_generate.py.

References is_definition_by_loc().

Referenced by get_serializable_classes_members().

def is_serializable_class_manual(node):
    for child in node.get_children():
        if child.spelling == 'cond_serialization_manual' and child.kind == clang.cindex.CursorKind.CXX_METHOD and not is_definition_by_loc(child):
            return True

    return False

def condformats_serialization_generate.log_flags	(		name,
			flags
	)

Definition at line 368 of file condformats_serialization_generate.py.

def log_flags(name, flags):
    logging.debug('%s = [', name)
    for flag in flags:
        logging.debug('    %s', flag)
    logging.debug(']')

def condformats_serialization_generate.main

(

)

Definition at line 570 of file condformats_serialization_generate.py.

References models.generate().

def main():
    parser = argparse.ArgumentParser(description='CMS Condition DB Serialization generator.')
    parser.add_argument('--verbose', '-v', action='count', help='Verbosity level. -v reports debugging information.')
    parser.add_argument('--output' , '-o', action='store', help='Specifies the path to the output file written. Default: src/Serialization.cc')
    parser.add_argument('--package', '-p', action='store', help='Specifies the path to the package to be processed. Default: the actual package')

    opts, args = parser.parse_known_args()

    logLevel = logging.INFO
    if opts.verbose < 1 and opts.output and opts.package:   # assume we're called by scram and reduce logging - but only if no verbose is requested
        logLevel = logging.WARNING

    if opts.verbose >= 1: 
        logLevel = logging.DEBUG

    logging.basicConfig(
        format = '[%(asctime)s] %(levelname)s: %(message)s',
        level = logLevel,
    )

    if opts.package:  # we got a directory name to process, assume it's from scram and remove the last ('/src') dir from the path
        pkgDir = opts.package
        if pkgDir.endswith('/src') :
            pkgDir, srcDir = os.path.split( opts.package )
        os.chdir( pkgDir )
        logging.info("Processing package in %s " % pkgDir)

    if opts.output:
        logging.info("Writing serialization code to %s " % opts.output)

    SerializationCodeGenerator( scramFlags=args[1:] ).generate( opts.output )

def condformats_serialization_generate.sanitise

(

var

)

Definition at line 415 of file condformats_serialization_generate.py.

Referenced by condformats_serialization_generate.SerializationCodeGenerator.generate().

def sanitise(var):
    return re.sub('[^a-zA-Z0-9.,-:]', '-', var)

def condformats_serialization_generate.split_path

(

path

)

Definition at line 324 of file condformats_serialization_generate.py.

def split_path(path):
    folders = []

    while True:
        path, folder = os.path.split(path)

        if folder != '':
            folders.append(folder)
        else:
            if path != '':
                folders.append(path)
            break

    folders.reverse()

    return folders