CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
condformats_serialization_generate Namespace Reference

Classes

class  SerializationCodeGenerator
 

Functions

def get_basic_type_string
 
def get_default_gcc_search_paths
 
def get_diagnostics
 
def get_flags
 
def get_serializable_classes_members
 
def get_statement
 
def get_type_string
 
def is_definition_by_loc
 
def is_serializable_class
 
def is_serializable_class_manual
 
def log_flags
 
def main
 
def sanitise
 
def split_path
 

Variables

string __author__ = 'Miguel Ojeda'
 
string __copyright__ = 'Copyright 2014, CERN'
 
list __credits__ = ['Giacomo Govi', 'Miguel Ojeda', 'Andreas Pfeiffer']
 
string __email__ = 'mojedasa@cern.ch'
 
string __license__ = 'Unknown'
 
string __maintainer__ = 'Miguel Ojeda'
 
string headers_template
 
string instantiation_template
 
string serialize_method_base_object_template = ' ar & boost::serialization::make_nvp("{base_object_name_sanitised}", boost::serialization::base_object<{base_object_name}>(*this));'
 
string serialize_method_begin_template
 
string serialize_method_end
 
string serialize_method_member_template = ''' ar & boost::serialization::make_nvp("{member_name_sanitised}", {member_name});'''
 
tuple skip_namespaces
 

Function Documentation

def condformats_serialization_generate.get_basic_type_string (   node)

Definition at line 120 of file condformats_serialization_generate.py.

Referenced by get_serializable_classes_members(), and get_type_string().

120 
121 def get_basic_type_string(node):
122  typekinds = {
123  clang.cindex.TypeKind.BOOL: 'bool',
124  clang.cindex.TypeKind.INT: 'int',
125  clang.cindex.TypeKind.LONG: 'long',
126  clang.cindex.TypeKind.UINT: 'unsigned int',
127  clang.cindex.TypeKind.ULONG: 'unsigned long',
128  clang.cindex.TypeKind.FLOAT: 'float',
129  clang.cindex.TypeKind.DOUBLE: 'double',
130  }
131 
132  if node.type.kind not in typekinds:
133  raise Exception('Not a known basic type.')
134 
135  return typekinds[node.type.kind]
136 
condformats_serialization_generate.get_basic_type_string
def get_basic_type_string
Definition: condformats_serialization_generate.py:120
def condformats_serialization_generate.get_default_gcc_search_paths (   gcc = 'g++',
  language = 'c++' 
)

Definition at line 360 of file condformats_serialization_generate.py.

360 
361 def get_default_gcc_search_paths(gcc = 'g++', language = 'c++'):
362  command = 'echo "" | %s -x%s -v -E - 2>&1' % (gcc, language)
363  logging.debug('Running: %s', command)
364 
365  paths = []
366  in_list = False
367  for line in subprocess.check_output(command, shell=True).splitlines():
368  if in_list:
369  if line == 'End of search list.':
370  break
371 
372  path = os.path.normpath(line.strip())
373 
374  # Intrinsics not handled by clang
375  # Note that /lib/gcc is found in other paths if not normalized,
376  # so has to go after normpath()
377  if '/lib/gcc/' in path:
378  continue
379 
380  paths.append('-I%s' % path)
381 
382  else:
383  if line == '#include <...> search starts here:':
384  in_list = True
385 
386  if not in_list:
387  raise Exception('Default GCC search paths not found.')
388 
389  return paths
condformats_serialization_generate.get_default_gcc_search_paths
def get_default_gcc_search_paths
Definition: condformats_serialization_generate.py:360
def condformats_serialization_generate.get_diagnostics (   translation_unit)

Definition at line 350 of file condformats_serialization_generate.py.

References python.multivaluedict.map().

350 
351 def get_diagnostics(translation_unit):
352  return map(lambda diag: {
353  'severity' : diag.severity,
354  'location' : diag.location,
355  'spelling' : diag.spelling,
356  'ranges' : diag.ranges,
357  'fixits' : diag.fixits,
358  }, translation_unit.diagnostics)
359 
condformats_serialization_generate.get_diagnostics
def get_diagnostics
Definition: condformats_serialization_generate.py:350
def condformats_serialization_generate.get_flags (   product_name,
  flags 
)

Definition at line 338 of file condformats_serialization_generate.py.

338 
339 def get_flags(product_name, flags):
340  command = "scram b echo_%s_%s | tail -1 | cut -d '=' -f '2-' | xargs -n1" % (product_name, flags)
341  logging.debug('Running: %s', command)
342  return subprocess.check_output(command, shell=True).splitlines()
condformats_serialization_generate.get_flags
def get_flags
Definition: condformats_serialization_generate.py:338
def condformats_serialization_generate.get_serializable_classes_members (   node,
  all_template_types = None,
  namespace = '',
  only_from_path = None 
)

Definition at line 145 of file condformats_serialization_generate.py.

References get_basic_type_string(), get_statement(), get_type_string(), is_definition_by_loc(), is_serializable_class(), is_serializable_class_manual(), and join().

145 
146 def get_serializable_classes_members(node, all_template_types=None, namespace='', only_from_path=None):
147  if all_template_types is None:
148  all_template_types = []
149 
150  logging.debug('%s', (node.spelling, all_template_types, namespace))
151  results = {}
152  for child in node.get_children():
153  if child.kind == clang.cindex.CursorKind.NAMESPACE:
154  # If we are in the root namespace, let's skip some common, big
155  # namespaces to improve speed and avoid serializing those.
156  if namespace == '':
157  if child.spelling in skip_namespaces:
158  continue
159 
160  # This skips compiler-specific stuff as well (e.g. __gnucxx...)
161  if child.spelling.startswith('_'):
162  continue
163 
164  logging.debug('Going into namespace %s', child.spelling)
165 
166  results.update(get_serializable_classes_members(child, all_template_types, namespace + child.spelling + '::', only_from_path))
167  continue
168 
169  if child.kind in [clang.cindex.CursorKind.CLASS_DECL, clang.cindex.CursorKind.STRUCT_DECL, clang.cindex.CursorKind.CLASS_TEMPLATE] and is_definition_by_loc(child):
170  logging.debug('Found struct/class/template definition: %s', child.spelling if child.spelling else '<anonymous>')
171 
172  if only_from_path is not None \
173  and child.location.file is not None \
174  and not child.location.file.name.startswith(only_from_path):
175  logging.debug('Skipping since it is an external of this package: %s', child.spelling)
176  continue
177 
178  serializable = is_serializable_class(child)
179  if serializable:
180  if child.spelling == '':
181  raise Exception('It is not possible to serialize anonymous/unnamed structs/classes.')
182 
183  if is_serializable_class_manual(child):
184  logging.info('Found manual serializable struct/class/template: %s', child.spelling)
185  continue
186 
187  logging.info('Found serializable struct/class/template: %s', child.spelling)
188 
189  template_types = []
190  base_objects = []
191  members = []
192  transients = []
193  after_serialize = False
194  after_serialize_count = 0
195  for member in child.get_children():
196  if after_serialize:
197  if after_serialize_count == 2:
198  after_serialize = False
199  else:
200  after_serialize_count = after_serialize_count + 1
201 
202  if member.kind != clang.cindex.CursorKind.UNEXPOSED_DECL:
203  raise Exception('Expected unexposed declaration (friend) after serialize() but found something else: looks like the COND_SERIALIZABLE macro has been changed without updating the script.')
204 
205  if 'COND_SERIALIZABLE' not in get_statement(member):
206  raise Exception('Could not find COND_SERIALIZABLE in the statement of the expected unexposed declarations (friends) after serialize(). Please fix the script/macro.')
207 
208  logging.debug('Skipping expected unexposed declaration (friend) after serialize().')
209  continue
210 
211  # Template type parameters (e.g. <typename T>)
212  if member.kind == clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER:
213  logging.info(' Found template type parameter: %s', member.spelling)
214  template_types.append(('typename', member.spelling))
215 
216  # Template non-type parameters (e.g. <int N>)
217  elif member.kind == clang.cindex.CursorKind.TEMPLATE_NON_TYPE_PARAMETER:
218  type_string = get_type_string(member)
219  if not type_string:
220  type_string = get_basic_type_string(member)
221  logging.info(' Found template non-type parameter: %s %s', type_string, member.spelling)
222  template_types.append((type_string, member.spelling))
223 
224  # Base objects
225  elif member.kind == clang.cindex.CursorKind.CXX_BASE_SPECIFIER:
226  # FIXME: .displayname gives sometimes things like "class mybase"
227  base_object = member.displayname
228  prefix = 'class '
229  if base_object.startswith(prefix):
230  base_object = base_object[len(prefix):]
231  logging.info(' Found base object: %s', base_object)
232  base_objects.append(base_object)
233 
234  # Member variables
235  elif member.kind == clang.cindex.CursorKind.FIELD_DECL and is_definition_by_loc(member):
236  # While clang 3.3 does not ignore unrecognized attributes
237  # (see http://llvm.org/viewvc/llvm-project?revision=165082&view=revision )
238  # for some reason they do not appear in the bindings yet
239  # so we just do it ourselves.
240 
241  # FIXME: To simplify and avoid parsing C++ ourselves, our transient
242  # attribute applies to *all* the variables declared in the same statement.
243  if 'COND_TRANSIENT' not in get_statement(member):
244  logging.info(' Found member variable: %s', member.spelling)
245  members.append(member.spelling)
246  else:
247  if serializable:
248  logging.info(' Found transient member variable: %s', member.spelling)
249  transients.append(member.spelling)
250  else:
251  raise Exception('Transient %s found for non-serializable class %s', member.spelling, child.spelling)
252 
253  elif member.kind == clang.cindex.CursorKind.FUNCTION_TEMPLATE and member.spelling == 'serialize':
254  after_serialize = True
255  logging.debug('Found serialize() method, skipping next two children which must be unexposed declarations.')
256 
257  elif member.kind in frozenset([
258  # For safety, we list all known kinds that we need to skip
259  # and raise in unknown cases (this helps catching problems
260  # with undefined classes)
261  clang.cindex.CursorKind.CONSTRUCTOR,
262  clang.cindex.CursorKind.DESTRUCTOR,
263  clang.cindex.CursorKind.CXX_METHOD,
264  clang.cindex.CursorKind.CXX_ACCESS_SPEC_DECL,
265  clang.cindex.CursorKind.FUNCTION_TEMPLATE,
266  clang.cindex.CursorKind.TYPEDEF_DECL,
267  clang.cindex.CursorKind.CLASS_DECL,
268  clang.cindex.CursorKind.ENUM_DECL,
269  clang.cindex.CursorKind.VAR_DECL,
270  clang.cindex.CursorKind.STRUCT_DECL,
271  clang.cindex.CursorKind.UNION_DECL,
272  clang.cindex.CursorKind.CONVERSION_FUNCTION,
273  clang.cindex.CursorKind.TYPE_REF,
274  clang.cindex.CursorKind.DECL_REF_EXPR,
275  ]):
276  logging.debug('Skipping member: %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind)
277 
278  elif member.kind == clang.cindex.CursorKind.UNEXPOSED_DECL:
279  statement = get_statement(member)
280 
281  # Friends are unexposed but they are not data to serialize
282  if 'friend' in statement:
283  # If we know about them, skip the warning
284  if \
285  'friend class ' in statement or \
286  'friend struct ' in statement or \
287  'friend std::ostream& operator<<(' in statement or \
288  'friend std::istream& operator>>(' in statement:
289  logging.debug('Skipping known friend: %s', statement.splitlines()[0])
290  continue
291 
292  # Otherwise warn
293  logging.warning('Unexposed declaration that looks like a friend declaration -- please check: %s %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind, statement)
294  continue
295 
296  raise Exception('Unexposed declaration. This probably means (at the time of writing) that an unknown class was found (may happen, for instance, when the compiler does not find the headers for std::vector, i.e. missing -I option): %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))
297 
298  else:
299  raise Exception('Unknown kind. Please fix the script: %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))
300 
301  if template_types:
302  template_use = '%s<%s>' % (child.spelling, ', '.join([template_type_name for (_, template_type_name) in template_types]))
303  else:
304  template_use = child.spelling
305 
306  new_namespace = namespace + template_use
307 
308  new_all_template_types = all_template_types + [template_types]
309 
310  results[new_namespace] = (child, serializable, new_all_template_types, base_objects, members, transients)
311 
312  results.update(get_serializable_classes_members(child, new_all_template_types, new_namespace + '::', only_from_path))
313 
314  for (klass, (node, serializable, all_template_types, base_objects, members, transients)) in results.items():
315  if serializable and len(members) == 0:
316  logging.info('No non-transient members found for serializable class %s', klass)
317 
318  return results
319 
condformats_serialization_generate.get_serializable_classes_members
def get_serializable_classes_members
Definition: condformats_serialization_generate.py:145
condformats_serialization_generate.get_statement
def get_statement
Definition: condformats_serialization_generate.py:103
condformats_serialization_generate.is_serializable_class_manual
def is_serializable_class_manual
Definition: condformats_serialization_generate.py:95
condformats_serialization_generate.is_serializable_class
def is_serializable_class
Definition: condformats_serialization_generate.py:78
condformats_serialization_generate.get_basic_type_string
def get_basic_type_string
Definition: condformats_serialization_generate.py:120
condformats_serialization_generate.get_type_string
def get_type_string
Definition: condformats_serialization_generate.py:137
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
condformats_serialization_generate.is_definition_by_loc
def is_definition_by_loc
Definition: condformats_serialization_generate.py:71
def condformats_serialization_generate.get_statement (   node)

Definition at line 103 of file condformats_serialization_generate.py.

Referenced by get_serializable_classes_members().

103 
104 def get_statement(node):
105  # For some cursor kinds, their location is empty (e.g. translation units
106  # and attributes); either because of a bug or because they do not have
107  # a meaningful 'start' -- however, the extent is always available
108  if node.extent.start.file is None:
109  return None
110 
111  filename = node.extent.start.file.name
112  start = node.extent.start.offset
113  end = node.extent.end.offset
114 
115  with open(filename, 'rb') as fd:
116  source = fd.read()
117 
118  return source[start:source.find(';', end)]
119 
condformats_serialization_generate.get_statement
def get_statement
Definition: condformats_serialization_generate.py:103
def condformats_serialization_generate.get_type_string (   node)

Definition at line 137 of file condformats_serialization_generate.py.

References get_basic_type_string().

Referenced by get_serializable_classes_members().

137 
138 def get_type_string(node):
139  spelling = node.type.get_declaration().spelling
140  if spelling is not None:
141  return spelling
142 
143  return get_basic_type_string(node)
144 
condformats_serialization_generate.get_basic_type_string
def get_basic_type_string
Definition: condformats_serialization_generate.py:120
condformats_serialization_generate.get_type_string
def get_type_string
Definition: condformats_serialization_generate.py:137
def condformats_serialization_generate.is_definition_by_loc (   node)

Definition at line 71 of file condformats_serialization_generate.py.

Referenced by get_serializable_classes_members(), is_serializable_class(), and is_serializable_class_manual().

71 
72 def is_definition_by_loc(node):
73  if node.get_definition() is None:
74  return False
75  if node.location is None or node.get_definition().location is None:
76  return False
77  return node.location == node.get_definition().location
condformats_serialization_generate.is_definition_by_loc
def is_definition_by_loc
Definition: condformats_serialization_generate.py:71
def condformats_serialization_generate.is_serializable_class (   node)

Definition at line 78 of file condformats_serialization_generate.py.

References is_definition_by_loc().

Referenced by get_serializable_classes_members().

78 
79 def is_serializable_class(node):
80  for child in node.get_children():
81  if child.spelling != 'serialize' or child.kind != clang.cindex.CursorKind.FUNCTION_TEMPLATE or is_definition_by_loc(child):
82  continue
83 
84  if [(x.spelling, x.kind, is_definition_by_loc(x), x.type.kind) for x in child.get_children()] != [
85  ('Archive', clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER, True, clang.cindex.TypeKind.UNEXPOSED),
86  ('ar', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.LVALUEREFERENCE),
87  ('version', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.UINT),
88  ]:
89  continue
90 
91  return True
92 
93  return False
94 
condformats_serialization_generate.is_serializable_class
def is_serializable_class
Definition: condformats_serialization_generate.py:78
condformats_serialization_generate.is_definition_by_loc
def is_definition_by_loc
Definition: condformats_serialization_generate.py:71
def condformats_serialization_generate.is_serializable_class_manual (   node)

Definition at line 95 of file condformats_serialization_generate.py.

References is_definition_by_loc().

Referenced by get_serializable_classes_members().

95 
96 def is_serializable_class_manual(node):
97  for child in node.get_children():
98  if child.spelling == 'cond_serialization_manual' and child.kind == clang.cindex.CursorKind.CXX_METHOD and not is_definition_by_loc(child):
99  return True
100 
101  return False
102 
condformats_serialization_generate.is_serializable_class_manual
def is_serializable_class_manual
Definition: condformats_serialization_generate.py:95
condformats_serialization_generate.is_definition_by_loc
def is_definition_by_loc
Definition: condformats_serialization_generate.py:71
def condformats_serialization_generate.log_flags (   name,
  flags 
)

Definition at line 343 of file condformats_serialization_generate.py.

343 
344 def log_flags(name, flags):
345  logging.debug('%s = [', name)
346  for flag in flags:
347  logging.debug(' %s', flag)
348  logging.debug(']')
349 
condformats_serialization_generate.log_flags
def log_flags
Definition: condformats_serialization_generate.py:343
def condformats_serialization_generate.main ( )

Definition at line 534 of file condformats_serialization_generate.py.

References relval_generation_module.generate().

534 
535 def main():
536  parser = argparse.ArgumentParser(description='CMS Condition DB Serialization generator.')
537  parser.add_argument('--verbose', '-v', action='count', help='Verbosity level. -v reports debugging information.')
538  parser.add_argument('--output' , '-o', action='store', help='Specifies the path to the output file written. Default: src/Serialization.cc')
539  parser.add_argument('--package', '-p', action='store', help='Specifies the path to the package to be processed. Default: the actual package')
540 
541  opts, args = parser.parse_known_args()
542 
543  logLevel = logging.INFO
544  if opts.verbose < 1 and opts.output and opts.package: # assume we're called by scram and reduce logging - but only if no verbose is requested
545  logLevel = logging.WARNING
546 
547  if opts.verbose >= 1:
548  logLevel = logging.DEBUG
549 
550  logging.basicConfig(
551  format = '[%(asctime)s] %(levelname)s: %(message)s',
552  level = logLevel,
553  )
554 
555  if opts.package: # we got a directory name to process, assume it's from scram and remove the last ('/src') dir from the path
556  pkgDir = opts.package
557  if pkgDir.endswith('/src') :
558  pkgDir, srcDir = os.path.split( opts.package )
559  os.chdir( pkgDir )
560  logging.info("Processing package in %s " % pkgDir)
561 
562  if opts.output:
563  logging.info("Writing serialization code to %s " % opts.output)
564 
565  SerializationCodeGenerator( scramFlags=args[1:] ).generate( opts.output )
relval_generation_module.generate
def generate
Definition: relval_generation_module.py:27
condformats_serialization_generate.SerializationCodeGenerator
Definition: condformats_serialization_generate.py:394
condformats_serialization_generate.main
def main
Definition: condformats_serialization_generate.py:534
def condformats_serialization_generate.sanitise (   var)

Definition at line 390 of file condformats_serialization_generate.py.

390 
391 def sanitise(var):
392  return re.sub('[^a-zA-Z0-9.,-:]', '-', var)
393 
condformats_serialization_generate.sanitise
def sanitise
Definition: condformats_serialization_generate.py:390
def condformats_serialization_generate.split_path (   path)

Definition at line 320 of file condformats_serialization_generate.py.

Referenced by init_filter().

320 
321 def split_path(path):
322  folders = []
323 
324  while True:
325  path, folder = os.path.split(path)
326 
327  if folder != '':
328  folders.append(folder)
329  else:
330  if path != '':
331  folders.append(path)
332  break
333 
334  folders.reverse()
335 
336  return folders
337 
condformats_serialization_generate.split_path
def split_path
Definition: condformats_serialization_generate.py:320

Variable Documentation

string condformats_serialization_generate.__author__ = 'Miguel Ojeda'

Definition at line 16 of file condformats_serialization_generate.py.

string condformats_serialization_generate.__copyright__ = 'Copyright 2014, CERN'

Definition at line 17 of file condformats_serialization_generate.py.

list condformats_serialization_generate.__credits__ = ['Giacomo Govi', 'Miguel Ojeda', 'Andreas Pfeiffer']

Definition at line 18 of file condformats_serialization_generate.py.

string condformats_serialization_generate.__email__ = 'mojedasa@cern.ch'

Definition at line 21 of file condformats_serialization_generate.py.

string condformats_serialization_generate.__license__ = 'Unknown'

Definition at line 19 of file condformats_serialization_generate.py.

string condformats_serialization_generate.__maintainer__ = 'Miguel Ojeda'

Definition at line 20 of file condformats_serialization_generate.py.

string condformats_serialization_generate.headers_template
Initial value:
1 = '''
2 #include "{headers}"
3 
4 #include <boost/serialization/base_object.hpp>
5 #include <boost/serialization/nvp.hpp>
6 #include <boost/serialization/export.hpp>
7 
8 #include "CondFormats/Serialization/interface/Equal.h"
9 #include "CondFormats/Serialization/interface/Instantiate.h"
10 
11 '''

Definition at line 33 of file condformats_serialization_generate.py.

string condformats_serialization_generate.instantiation_template
Initial value:
1 = '''COND_SERIALIZATION_INSTANTIATE({klass});
2 '''

Definition at line 56 of file condformats_serialization_generate.py.

string condformats_serialization_generate.serialize_method_base_object_template = ' ar & boost::serialization::make_nvp("{base_object_name_sanitised}", boost::serialization::base_object<{base_object_name}>(*this));'

Definition at line 49 of file condformats_serialization_generate.py.

string condformats_serialization_generate.serialize_method_begin_template
Initial value:
1 = '''template <class Archive>
2 void {klass}::serialize(Archive & ar, const unsigned int)
3 {{'''

Definition at line 45 of file condformats_serialization_generate.py.

string condformats_serialization_generate.serialize_method_end
Initial value:
1 = '''}
2 '''

Definition at line 53 of file condformats_serialization_generate.py.

string condformats_serialization_generate.serialize_method_member_template = ''' ar & boost::serialization::make_nvp("{member_name_sanitised}", {member_name});'''

Definition at line 51 of file condformats_serialization_generate.py.

tuple condformats_serialization_generate.skip_namespaces
Initial value:
1 = frozenset([
2  # Do not go inside anonymous namespaces (static)
3  '',
4 
5  # Do not go inside some standard namespaces
6  'std', 'boost', 'mpl_', 'boost_swap_impl',
7 
8  # Do not go inside some big namespaces coming from externals
9  'ROOT', 'edm', 'ora', 'coral', 'CLHEP', 'Geom', 'HepGeom',
10 ])

Definition at line 60 of file condformats_serialization_generate.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5