2 '''CMS Conditions DB Serialization generator.
4 Generates the non-intrusive serialization code required for the classes
5 marked with the COND_SERIALIZABLE macro.
7 The code was taken from the prototype that did many other things as well
8 (finding transients, marking serializable classes, etc.). After removing
9 everything but what is required to build the serialization, the code was
10 made more robust and cleaned up a bit to be integrated on the BoostIO IB.
11 However, the code still needs to be restructured a bit more to improve
12 readability (e.g. name some constants, use a template engine, ask for
13 clang's bindings to be installed along clang itself, etc.).
16 __author__ =
'Miguel Ojeda'
17 __copyright__ =
'Copyright 2014, CERN'
18 __credits__ = [
'Giacomo Govi',
'Miguel Ojeda',
'Andreas Pfeiffer']
19 __license__ =
'Unknown'
20 __maintainer__ =
'Miguel Ojeda'
21 __email__ =
'mojedasa@cern.ch'
33 headers_template =
'''
36 #include <boost/serialization/base_object.hpp>
37 #include <boost/serialization/nvp.hpp>
38 #include <boost/serialization/export.hpp>
40 #include "CondFormats/Serialization/interface/Equal.h"
41 #include "CondFormats/Serialization/interface/Instantiate.h"
45 serialize_method_begin_template =
'''template <class Archive>
46 void {klass}::serialize(Archive & ar, const unsigned int)
49 serialize_method_base_object_template =
' ar & boost::serialization::make_nvp("{base_object_name_sanitised}", boost::serialization::base_object<{base_object_name}>(*this));'
51 serialize_method_member_template =
''' ar & boost::serialization::make_nvp("{member_name_sanitised}", {member_name});'''
53 serialize_method_end =
'''}
56 instantiation_template =
'''COND_SERIALIZATION_INSTANTIATE({klass});
60 skip_namespaces = frozenset([
65 'std',
'boost',
'mpl_',
'boost_swap_impl',
68 'ROOT',
'edm',
'ora',
'coral',
'CLHEP',
'Geom',
'HepGeom',
72 if node.get_definition()
is None:
74 if node.location
is None or node.get_definition().location
is None:
76 return node.location == node.get_definition().location
79 for child
in node.get_children():
80 if child.spelling !=
'serialize' or child.kind != clang.cindex.CursorKind.FUNCTION_TEMPLATE
or is_definition_by_loc(child):
83 if [(x.spelling, x.kind,
is_definition_by_loc(x), x.type.kind)
for x
in child.get_children()] != [
84 (
'Archive', clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER,
True, clang.cindex.TypeKind.UNEXPOSED),
85 (
'ar', clang.cindex.CursorKind.PARM_DECL,
True, clang.cindex.TypeKind.LVALUEREFERENCE),
86 (
'version', clang.cindex.CursorKind.PARM_DECL,
True, clang.cindex.TypeKind.UINT),
96 for child
in node.get_children():
97 if child.spelling ==
'cond_serialization_manual' and child.kind == clang.cindex.CursorKind.CXX_METHOD
and not is_definition_by_loc(child):
107 if node.extent.start.file
is None:
110 filename = node.extent.start.file.name
111 start = node.extent.start.offset
112 end = node.extent.end.offset
114 with open(filename,
'rb')
as fd:
117 return source[start:source.find(
';', end)]
122 clang.cindex.TypeKind.BOOL:
'bool',
123 clang.cindex.TypeKind.INT:
'int',
124 clang.cindex.TypeKind.LONG:
'long',
125 clang.cindex.TypeKind.UINT:
'unsigned int',
126 clang.cindex.TypeKind.ULONG:
'unsigned long',
127 clang.cindex.TypeKind.FLOAT:
'float',
128 clang.cindex.TypeKind.DOUBLE:
'double',
131 if node.type.kind
not in typekinds:
132 raise Exception(
'Not a known basic type.')
134 return typekinds[node.type.kind]
138 spelling = node.type.get_declaration().spelling
139 if spelling
is not None:
146 if all_template_types
is None:
147 all_template_types = []
149 logging.debug(
'%s', (node.spelling, all_template_types, namespace))
151 for child
in node.get_children():
152 if child.kind == clang.cindex.CursorKind.NAMESPACE:
156 if child.spelling
in skip_namespaces:
160 if child.spelling.startswith(
'_'):
163 logging.debug(
'Going into namespace %s', child.spelling)
168 if child.kind
in [clang.cindex.CursorKind.CLASS_DECL, clang.cindex.CursorKind.STRUCT_DECL, clang.cindex.CursorKind.CLASS_TEMPLATE]
and is_definition_by_loc(child):
169 logging.debug(
'Found struct/class/template definition: %s', child.spelling
if child.spelling
else '<anonymous>')
171 if only_from_path
is not None \
172 and child.location.file
is not None \
173 and not child.location.file.name.startswith(only_from_path):
174 logging.debug(
'Skipping since it is an external of this package: %s', child.spelling)
179 if child.spelling ==
'':
180 raise Exception(
'It is not possible to serialize anonymous/unnamed structs/classes.')
183 logging.info(
'Found manual serializable struct/class/template: %s', child.spelling)
186 logging.info(
'Found serializable struct/class/template: %s', child.spelling)
192 after_serialize =
False
193 after_serialize_count = 0
194 for member
in child.get_children():
196 if after_serialize_count == 2:
197 after_serialize =
False
199 after_serialize_count = after_serialize_count + 1
201 if member.kind != clang.cindex.CursorKind.UNEXPOSED_DECL:
202 raise Exception(
'Expected unexposed declaration (friend) after serialize() but found something else: looks like the COND_SERIALIZABLE macro has been changed without updating the script.')
205 raise Exception(
'Could not find COND_SERIALIZABLE in the statement of the expected unexposed declarations (friends) after serialize(). Please fix the script/macro.')
207 logging.debug(
'Skipping expected unexposed declaration (friend) after serialize().')
211 if member.kind == clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER:
212 logging.info(
' Found template type parameter: %s', member.spelling)
213 template_types.append((
'typename', member.spelling))
216 elif member.kind == clang.cindex.CursorKind.TEMPLATE_NON_TYPE_PARAMETER:
220 logging.info(
' Found template non-type parameter: %s %s', type_string, member.spelling)
221 template_types.append((type_string, member.spelling))
224 elif member.kind == clang.cindex.CursorKind.CXX_BASE_SPECIFIER:
226 base_object = member.displayname
228 if base_object.startswith(prefix):
229 base_object = base_object[len(prefix):]
230 logging.info(
' Found base object: %s', base_object)
231 base_objects.append(base_object)
243 logging.info(
' Found member variable: %s', member.spelling)
244 members.append(member.spelling)
247 logging.info(
' Found transient member variable: %s', member.spelling)
248 transients.append(member.spelling)
250 raise Exception(
'Transient %s found for non-serializable class %s', member.spelling, child.spelling)
252 elif member.kind == clang.cindex.CursorKind.FUNCTION_TEMPLATE
and member.spelling ==
'serialize':
253 after_serialize =
True
254 logging.debug(
'Found serialize() method, skipping next two children which must be unexposed declarations.')
256 elif member.kind
in frozenset([
260 clang.cindex.CursorKind.CONSTRUCTOR,
261 clang.cindex.CursorKind.DESTRUCTOR,
262 clang.cindex.CursorKind.CXX_METHOD,
263 clang.cindex.CursorKind.CXX_ACCESS_SPEC_DECL,
264 clang.cindex.CursorKind.FUNCTION_TEMPLATE,
265 clang.cindex.CursorKind.TYPEDEF_DECL,
266 clang.cindex.CursorKind.CLASS_DECL,
267 clang.cindex.CursorKind.ENUM_DECL,
268 clang.cindex.CursorKind.VAR_DECL,
269 clang.cindex.CursorKind.STRUCT_DECL,
270 clang.cindex.CursorKind.UNION_DECL,
271 clang.cindex.CursorKind.CONVERSION_FUNCTION,
272 clang.cindex.CursorKind.TYPE_REF,
273 clang.cindex.CursorKind.DECL_REF_EXPR,
275 logging.debug(
'Skipping member: %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind)
277 elif member.kind == clang.cindex.CursorKind.UNEXPOSED_DECL:
281 if 'friend' in statement:
284 'friend class ' in statement
or \
285 'friend struct ' in statement
or \
286 'friend std::ostream& operator<<(' in statement
or \
287 'friend std::istream& operator>>(' in statement:
288 logging.debug(
'Skipping known friend: %s', statement.splitlines()[0])
292 logging.warning(
'Unexposed declaration that looks like a friend declaration -- please check: %s %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind, statement)
295 raise Exception(
'Unexposed declaration. This probably means (at the time of writing) that an unknown class was found (may happen, for instance, when the compiler does not find the headers for std::vector, i.e. missing -I option): %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))
298 raise Exception(
'Unknown kind. Please fix the script: %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))
301 template_use =
'%s<%s>' % (child.spelling,
', '.
join([template_type_name
for (_, template_type_name)
in template_types]))
303 template_use = child.spelling
305 new_namespace = namespace + template_use
307 new_all_template_types = all_template_types + [template_types]
309 results[new_namespace] = (child, serializable, new_all_template_types, base_objects, members, transients)
313 for (klass, (node, serializable, all_template_types, base_objects, members, transients))
in results.items():
314 if serializable
and len(members) == 0:
315 logging.info(
'No non-transient members found for serializable class %s', klass)
324 path, folder = os.path.split(path)
327 folders.append(folder)
339 command =
"scram b echo_%s_%s | tail -1 | cut -d '=' -f '2-' | xargs -n1" % (product_name, flags)
340 logging.debug(
'Running: %s', command)
341 return subprocess.check_output(command, shell=
True).splitlines()
344 logging.debug(
'%s = [', name)
346 logging.debug(
' %s', flag)
351 return map(
lambda diag: {
352 'severity' : diag.severity,
353 'location' : diag.location,
354 'spelling' : diag.spelling,
355 'ranges' : diag.ranges,
356 'fixits' : diag.fixits,
357 }, translation_unit.diagnostics)
361 command =
'echo "" | %s -x%s -v -E - 2>&1' % (gcc, language)
362 logging.debug(
'Running: %s', command)
366 for line
in subprocess.check_output(command, shell=
True).splitlines():
368 if line ==
'End of search list.':
371 path = os.path.normpath(line.strip())
376 if '/lib/gcc/' in path:
379 paths.append(
'-I%s' % path)
382 if line ==
'#include <...> search starts here:':
386 raise Exception(
'Default GCC search paths not found.')
391 return re.sub(
'[^a-zA-Z0-9.,-:]',
'-', var)
400 raise Exception(
'CMSSW_BASE is not set.')
401 logging.debug(
'cmssw_base = %s', self.
cmssw_base)
404 logging.debug(
'cwd = %s', cwd)
407 raise Exception(
'The filepath does not start with CMSSW_BASE.')
410 logging.debug(
'relative_path = %s', relative_path)
413 logging.debug(
'splitpath = %s', self.
split_path)
416 raise Exception(
'This script requires to be run inside a CMSSW package (usually within CondFormats), e.g. CondFormats/Alignment. The current path is: %s' % self.
split_path)
419 raise Exception(
'The first folder should be src.')
422 raise Exception(
'The second folder should be CondFormats.')
425 logging.debug(
'product_name = %s', product_name)
440 flags = [
'-xc++'] + cpp_flags + cxx_flags + std_flags
443 logging.debug(
'headers_h = %s', headers_h)
444 if not os.path.exists(headers_h):
445 raise Exception(
'File %s does not exist. Impossible to serialize package.' % headers_h)
447 logging.info(
'Searching serializable classes in %s/%s ...', self.
split_path[1], self.
split_path[2])
449 logging.debug(
'Parsing C++ classes in file %s ...', headers_h)
450 index = clang.cindex.Index.create()
451 translation_unit = index.parse(headers_h, flags)
452 if not translation_unit:
455 severity_names = (
'Ignored',
'Note',
'Warning',
'Error',
'Fatal')
456 get_severity_name =
lambda severity_num: severity_names[severity_num]
if severity_num < len(severity_names)
else 'Unknown'
457 max_severity_level = 0
459 for diagnostic
in diagnostics:
463 if diagnostic[
'spelling'].startswith(
'argument unused during compilation') \
464 or diagnostic[
'spelling'].startswith(
'unknown warning option'):
468 logf(
' at line %s in %s', diagnostic[
'location'].line, diagnostic[
'location'].file)
470 max_severity_level =
max(max_severity_level, diagnostic[
'severity'])
472 if max_severity_level >= 3:
473 raise Exception(
'Please, resolve all errors before proceeding.')
481 flags = [ flag
for flag
in flagsIn
if not flag.startswith((
'-march',
'-mtune',
'-fdebug-prefix-map')) ]
482 blackList = [
'--',
'-fipa-pta']
483 return [x
for x
in flags
if x
not in blackList]
487 filename = outFileName
491 n_serializable_classes = 0
493 source = headers_template.format(headers=os.path.join(self.
split_path[1], self.
split_path[2],
'src',
'headers.h'))
495 for klass
in sorted(self.classes):
496 (node, serializable, all_template_types, base_objects, members, transients) = self.classes[klass]
501 n_serializable_classes += 1
503 skip_instantiation =
False
504 for template_types
in all_template_types:
506 skip_instantiation =
True
507 source += (
'template <%s>' %
', '.
join([
'%s %s' % template_type
for template_type
in template_types])) +
'\n'
509 source += serialize_method_begin_template.format(klass=klass) +
'\n'
511 for base_object_name
in base_objects:
512 base_object_name_sanitised =
sanitise(base_object_name)
513 source += serialize_method_base_object_template.format(base_object_name=base_object_name, base_object_name_sanitised=base_object_name_sanitised) +
'\n'
515 for member_name
in members:
517 source += serialize_method_member_template.format(member_name=member_name, member_name_sanitised=member_name_sanitised) +
'\n'
519 source += serialize_method_end
521 if skip_instantiation:
524 source += instantiation_template.format(klass=klass) +
'\n'
526 if n_serializable_classes == 0:
527 raise Exception(
'No serializable classes found, while this package has a headers.h file.')
530 if os.path.exists(
'./src/SerializationManual.h' ) :
531 source +=
'#include "%s/%s/src/SerializationManual.h"\n' % (self.
split_path[1], self.
split_path[2])
533 logging.info(
'Writing serialization code for %s classes in %s ...', n_serializable_classes, filename)
534 with open(filename,
'wb')
as fd:
540 parser.add_argument(
'--verbose',
'-v', action=
'count', help=
'Verbosity level. -v reports debugging information.')
541 parser.add_argument(
'--output' ,
'-o', action=
'store', help=
'Specifies the path to the output file written. Default: src/Serialization.cc')
542 parser.add_argument(
'--package',
'-p', action=
'store', help=
'Specifies the path to the package to be processed. Default: the actual package')
544 opts, args = parser.parse_known_args()
546 logLevel = logging.INFO
547 if opts.verbose < 1
and opts.output
and opts.package:
548 logLevel = logging.WARNING
550 if opts.verbose >= 1:
551 logLevel = logging.DEBUG
554 format =
'[%(asctime)s] %(levelname)s: %(message)s',
559 pkgDir = opts.package
560 if pkgDir.endswith(
'/src') :
561 pkgDir, srcDir = os.path.split( opts.package )
563 logging.info(
"Processing package in %s " % pkgDir)
566 logging.info(
"Writing serialization code to %s " % opts.output)
570 if __name__ ==
'__main__':
static std::string join(char **cmd)