CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Classes | Functions | Variables
condformats_serialization_generate Namespace Reference

Classes

class  SerializationCodeGenerator
 

Functions

def get_basic_type_string
 
def get_clang_version
 
def get_default_gcc_search_paths
 
def get_diagnostics
 
def get_flags
 
def get_serializable_classes_members
 
def get_statement
 
def get_type_string
 
def is_definition_by_loc
 
def is_friend_decl
 
def is_serializable_class
 
def is_serializable_class_manual
 
def log_flags
 
def main
 
def sanitise
 
def split_path
 

Variables

string __author__ = 'Miguel Ojeda'
 
string __copyright__ = 'Copyright 2014, CERN'
 
list __credits__ = ['Giacomo Govi', 'Miguel Ojeda', 'Andreas Pfeiffer']
 
string __email__ = 'mojedasa@cern.ch'
 
string __license__ = 'Unknown'
 
string __maintainer__ = 'Miguel Ojeda'
 
 clang_version = None
 
string headers_template
 
string instantiation_template
 
string serialize_method_base_object_template = ' ar & boost::serialization::make_nvp("{base_object_name_sanitised}", boost::serialization::base_object<{base_object_name}>(*this));'
 
string serialize_method_begin_template
 
string serialize_method_end
 
string serialize_method_member_template = ''' ar & boost::serialization::make_nvp("{member_name_sanitised}", {member_name});'''
 
tuple skip_namespaces
 

Function Documentation

def condformats_serialization_generate.get_basic_type_string (   node)

Definition at line 122 of file condformats_serialization_generate.py.

Referenced by get_serializable_classes_members(), and get_type_string().

123 def get_basic_type_string(node):
124  typekinds = {
125  clang.cindex.TypeKind.BOOL: 'bool',
126  clang.cindex.TypeKind.INT: 'int',
127  clang.cindex.TypeKind.LONG: 'long',
128  clang.cindex.TypeKind.UINT: 'unsigned int',
129  clang.cindex.TypeKind.ULONG: 'unsigned long',
130  clang.cindex.TypeKind.FLOAT: 'float',
131  clang.cindex.TypeKind.DOUBLE: 'double',
132  }
133 
134  if node.type.kind not in typekinds:
135  raise Exception('Not a known basic type.')
136 
137  return typekinds[node.type.kind]
138 
def condformats_serialization_generate.get_clang_version ( )
Extract clang version and set global clang_version and also return the same value.

Definition at line 348 of file condformats_serialization_generate.py.

References edm.decode(), submitPVValidationJobs.split(), and digitizers_cfi.strip.

Referenced by is_friend_decl().

349 def get_clang_version():
350  """Extract clang version and set global clang_version and also return the same value."""
351  global clang_version
352  if clang_version is not None:
353  return clang_version
354  command = "clang --version | grep 'clang version' | sed 's/clang version//'"
355  logging.debug("Running: {0}".format(command))
356  (clang_version_major, clang_version_minor, clang_version_patchlevel) = subprocess.check_output(command, shell=True).splitlines()[0].decode('ascii').strip().split(" ")[0].split('.', 3)
357  clang_version = (int(clang_version_major), int(clang_version_minor), int(clang_version_patchlevel))
358  logging.debug("Detected Clang version: {0}".format(clang_version))
359  return clang_version
bool decode(bool &, std::string const &)
Definition: types.cc:71
def condformats_serialization_generate.get_default_gcc_search_paths (   gcc = 'g++',
  language = 'c++' 
)

Definition at line 386 of file condformats_serialization_generate.py.

387 def get_default_gcc_search_paths(gcc = 'g++', language = 'c++'):
388  command = 'echo "" | %s -x%s -v -E - 2>&1' % (gcc, language)
389  logging.debug('Running: %s', command)
390 
391  paths = []
392  in_list = False
393  for line in [l.decode("ascii") for l in subprocess.check_output(command, shell=True).splitlines()]:
394  if in_list:
395  if line == 'End of search list.':
396  break
397 
398  path = os.path.normpath(line.strip())
399 
400  # Intrinsics not handled by clang
401  # Note that /lib/gcc is found in other paths if not normalized,
402  # so has to go after normpath()
403  if '/lib/gcc/' in path:
404  continue
405 
406  paths.append('-I%s' % path)
407 
408  else:
409  if line == '#include <...> search starts here:':
410  in_list = True
411 
412  if not in_list:
413  raise Exception('Default GCC search paths not found.')
414 
415  return paths
def condformats_serialization_generate.get_diagnostics (   translation_unit)

Definition at line 376 of file condformats_serialization_generate.py.

377 def get_diagnostics(translation_unit):
378  return map(lambda diag: {
379  'severity' : diag.severity,
380  'location' : diag.location,
381  'spelling' : diag.spelling,
382  'ranges' : diag.ranges,
383  'fixits' : diag.fixits,
384  }, translation_unit.diagnostics)
385 
def condformats_serialization_generate.get_flags (   product_name,
  flags 
)

Definition at line 343 of file condformats_serialization_generate.py.

344 def get_flags(product_name, flags):
345  command = "scram b echo_%s_%s | tail -1 | cut -d '=' -f '2-' | xargs -n1" % (product_name, flags)
346  logging.debug('Running: %s', command)
347  return subprocess.check_output(command, shell=True).splitlines()
def condformats_serialization_generate.get_serializable_classes_members (   node,
  all_template_types = None,
  namespace = '',
  only_from_path = None 
)

Definition at line 147 of file condformats_serialization_generate.py.

References get_basic_type_string(), get_statement(), get_type_string(), is_definition_by_loc(), is_friend_decl(), is_serializable_class(), is_serializable_class_manual(), and join().

148 def get_serializable_classes_members(node, all_template_types=None, namespace='', only_from_path=None):
149  if all_template_types is None:
150  all_template_types = []
151 
152  logging.debug('%s', (node.spelling, all_template_types, namespace))
153  results = {}
154  for child in node.get_children():
155  if child.kind == clang.cindex.CursorKind.NAMESPACE:
156  # If we are in the root namespace, let's skip some common, big
157  # namespaces to improve speed and avoid serializing those.
158  if namespace == '':
159  if child.spelling in skip_namespaces:
160  continue
161 
162  # This skips compiler-specific stuff as well (e.g. __gnucxx...)
163  if child.spelling.startswith('_'):
164  continue
165 
166  logging.debug('Going into namespace %s', child.spelling)
167 
168  results.update(get_serializable_classes_members(child, all_template_types, namespace + child.spelling + '::', only_from_path))
169  continue
170 
171  if child.kind in [clang.cindex.CursorKind.CLASS_DECL, clang.cindex.CursorKind.STRUCT_DECL, clang.cindex.CursorKind.CLASS_TEMPLATE] and is_definition_by_loc(child):
172  logging.debug('Found struct/class/template definition: %s', child.spelling if child.spelling else '<anonymous>')
173 
174  if only_from_path is not None \
175  and child.location.file is not None \
176  and not child.location.file.name.startswith(only_from_path):
177  logging.debug('Skipping since it is an external of this package: %s', child.spelling)
178  continue
179 
180  serializable = is_serializable_class(child)
181  if serializable:
182  if child.spelling == '':
183  raise Exception('It is not possible to serialize anonymous/unnamed structs/classes.')
184 
186  logging.info('Found manual serializable struct/class/template: %s', child.spelling)
187  continue
188 
189  logging.info('Found serializable struct/class/template: %s', child.spelling)
190 
191  template_types = []
192  base_objects = []
193  members = []
194  transients = []
195  after_serialize = False
196  after_serialize_count = 0
197  for member in child.get_children():
198  if after_serialize:
199  if after_serialize_count == 2:
200  after_serialize = False
201  else:
202  after_serialize_count = after_serialize_count + 1
203 
204  if not is_friend_decl(member.kind):
205  raise Exception('Expected unexposed declaration (friend) after serialize() but found something else: looks like the COND_SERIALIZABLE macro has been changed without updating the script.')
206 
207  if 'COND_SERIALIZABLE' not in get_statement(member):
208  raise Exception('Could not find COND_SERIALIZABLE in the statement of the expected unexposed declarations (friends) after serialize(). Please fix the script/macro.')
209 
210  logging.debug('Skipping expected unexposed declaration (friend) after serialize().')
211  continue
212 
213  # Template type parameters (e.g. <typename T>)
214  if member.kind == clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER:
215  logging.info(' Found template type parameter: %s', member.spelling)
216  template_types.append(('typename', member.spelling))
217 
218  # Template non-type parameters (e.g. <int N>)
219  elif member.kind == clang.cindex.CursorKind.TEMPLATE_NON_TYPE_PARAMETER:
220  type_string = get_type_string(member)
221  if not type_string:
222  type_string = get_basic_type_string(member)
223  logging.info(' Found template non-type parameter: %s %s', type_string, member.spelling)
224  template_types.append((type_string, member.spelling))
225 
226  # Base objects
227  elif member.kind == clang.cindex.CursorKind.CXX_BASE_SPECIFIER:
228  # FIXME: .displayname gives sometimes things like "class mybase"
229  base_object = member.displayname
230  prefix = 'class '
231  if base_object.startswith(prefix):
232  base_object = base_object[len(prefix):]
233  logging.info(' Found base object: %s', base_object)
234  base_objects.append(base_object)
235 
236  # Member variables
237  elif member.kind == clang.cindex.CursorKind.FIELD_DECL and is_definition_by_loc(member):
238  # While clang 3.3 does not ignore unrecognized attributes
239  # (see http://llvm.org/viewvc/llvm-project?revision=165082&view=revision )
240  # for some reason they do not appear in the bindings yet
241  # so we just do it ourselves.
242 
243  # FIXME: To simplify and avoid parsing C++ ourselves, our transient
244  # attribute applies to *all* the variables declared in the same statement.
245  if 'COND_TRANSIENT' not in get_statement(member):
246  logging.info(' Found member variable: %s', member.spelling)
247  members.append(member.spelling)
248  else:
249  if serializable:
250  logging.info(' Found transient member variable: %s', member.spelling)
251  transients.append(member.spelling)
252  else:
253  raise Exception('Transient %s found for non-serializable class %s', member.spelling, child.spelling)
254 
255  elif member.kind == clang.cindex.CursorKind.FUNCTION_TEMPLATE and member.spelling == 'serialize':
256  after_serialize = True
257  logging.debug('Found serialize() method, skipping next two children which must be unexposed declarations.')
258 
259  elif member.kind in frozenset([
260  # For safety, we list all known kinds that we need to skip
261  # and raise in unknown cases (this helps catching problems
262  # with undefined classes)
263  clang.cindex.CursorKind.CONSTRUCTOR,
264  clang.cindex.CursorKind.DESTRUCTOR,
265  clang.cindex.CursorKind.CXX_METHOD,
266  clang.cindex.CursorKind.CXX_ACCESS_SPEC_DECL,
267  clang.cindex.CursorKind.FUNCTION_TEMPLATE,
268  clang.cindex.CursorKind.TYPEDEF_DECL,
269  clang.cindex.CursorKind.CLASS_DECL,
270  clang.cindex.CursorKind.ENUM_DECL,
271  clang.cindex.CursorKind.VAR_DECL,
272  clang.cindex.CursorKind.STRUCT_DECL,
273  clang.cindex.CursorKind.UNION_DECL,
274  clang.cindex.CursorKind.CONVERSION_FUNCTION,
275  clang.cindex.CursorKind.TYPE_REF,
276  clang.cindex.CursorKind.DECL_REF_EXPR,
277  clang.cindex.CursorKind.CLASS_TEMPLATE,
278  clang.cindex.CursorKind.TYPE_ALIAS_DECL,
279  ]):
280  logging.debug('Skipping member: %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind)
281 
282  elif is_friend_decl(member.kind):
283  statement = get_statement(member)
284 
285  # Friends are unexposed but they are not data to serialize
286  if 'friend' in statement:
287  # If we know about them, skip the warning
288  if \
289  'friend class ' in statement or \
290  'friend struct ' in statement or \
291  'friend std::ostream& operator<<(' in statement or \
292  'friend std::istream& operator>>(' in statement:
293  logging.debug('Skipping known friend: %s', statement.splitlines()[0])
294  continue
295 
296  # Otherwise warn
297  logging.warning('Unexposed declaration that looks like a friend declaration -- please check: %s %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind, statement)
298  continue
299 
300  raise Exception('Unexposed declaration. This probably means (at the time of writing) that an unknown class was found (may happen, for instance, when the compiler does not find the headers for std::vector, i.e. missing -I option): %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))
301 
302  else:
303  statement = get_statement(member)
304  raise Exception('Unknown kind. Please fix the script: %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))
305 
306  if template_types:
307  template_use = '%s<%s>' % (child.spelling, ', '.join([template_type_name for (_, template_type_name) in template_types]))
308  else:
309  template_use = child.spelling
310 
311  new_namespace = namespace + template_use
312 
313  new_all_template_types = all_template_types + [template_types]
314 
315  results[new_namespace] = (child, serializable, new_all_template_types, base_objects, members, transients)
316 
317  results.update(get_serializable_classes_members(child, new_all_template_types, new_namespace + '::', only_from_path))
318 
319  for (klass, (node, serializable, all_template_types, base_objects, members, transients)) in results.items():
320  if serializable and len(members) == 0:
321  logging.info('No non-transient members found for serializable class %s', klass)
322 
323  return results
324 
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def condformats_serialization_generate.get_statement (   node)

Definition at line 105 of file condformats_serialization_generate.py.

References edm.decode().

Referenced by get_serializable_classes_members().

106 def get_statement(node):
107  # For some cursor kinds, their location is empty (e.g. translation units
108  # and attributes); either because of a bug or because they do not have
109  # a meaningful 'start' -- however, the extent is always available
110  if node.extent.start.file is None:
111  return None
112 
113  filename = node.extent.start.file.name
114  start = node.extent.start.offset
115  end = node.extent.end.offset
116 
117  with open(filename, 'rb') as fd:
118  source = fd.read().decode('latin-1')
119 
120  return source[start:source.find(';', end)]
121 
bool decode(bool &, std::string const &)
Definition: types.cc:71
def condformats_serialization_generate.get_type_string (   node)

Definition at line 139 of file condformats_serialization_generate.py.

References get_basic_type_string().

Referenced by get_serializable_classes_members().

140 def get_type_string(node):
141  spelling = node.type.get_declaration().spelling
142  if spelling is not None:
143  return spelling
144 
145  return get_basic_type_string(node)
146 
def condformats_serialization_generate.is_definition_by_loc (   node)

Definition at line 73 of file condformats_serialization_generate.py.

Referenced by get_serializable_classes_members(), is_serializable_class(), and is_serializable_class_manual().

73 
74 def is_definition_by_loc(node):
75  if node.get_definition() is None:
76  return False
77  if node.location is None or node.get_definition().location is None:
78  return False
79  return node.location == node.get_definition().location
def condformats_serialization_generate.is_friend_decl (   memkind)
Check if declaration is a friend

Definition at line 360 of file condformats_serialization_generate.py.

References get_clang_version().

Referenced by get_serializable_classes_members().

361 def is_friend_decl(memkind):
362  """Check if declaration is a friend"""
363  clangv = get_clang_version()
364  if clangv >= (4, 0, 0):
365  return memkind == clang.cindex.CursorKind.FRIEND_DECL
366  else:
367  return memkind == clang.cindex.CursorKind.UNEXPOSED_DECL
368  return false
def condformats_serialization_generate.is_serializable_class (   node)

Definition at line 80 of file condformats_serialization_generate.py.

References is_definition_by_loc().

Referenced by get_serializable_classes_members().

80 
81 def is_serializable_class(node):
82  for child in node.get_children():
83  if child.spelling != 'serialize' or child.kind != clang.cindex.CursorKind.FUNCTION_TEMPLATE or is_definition_by_loc(child):
84  continue
85 
86  if [(x.spelling, x.kind, is_definition_by_loc(x), x.type.kind) for x in child.get_children()] != [
87  ('Archive', clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER, True, clang.cindex.TypeKind.UNEXPOSED),
88  ('ar', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.LVALUEREFERENCE),
89  ('version', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.UINT),
90  ]:
91  continue
92 
93  return True
94 
95  return False
96 
def condformats_serialization_generate.is_serializable_class_manual (   node)

Definition at line 97 of file condformats_serialization_generate.py.

References is_definition_by_loc().

Referenced by get_serializable_classes_members().

97 
99  for child in node.get_children():
100  if child.spelling == 'cond_serialization_manual' and child.kind == clang.cindex.CursorKind.CXX_METHOD and not is_definition_by_loc(child):
101  return True
102 
103  return False
104 
def condformats_serialization_generate.log_flags (   name,
  flags 
)

Definition at line 369 of file condformats_serialization_generate.py.

370 def log_flags(name, flags):
371  logging.debug('%s = [', name)
372  for flag in flags:
373  logging.debug(' %s', flag)
374  logging.debug(']')
375 
def condformats_serialization_generate.main ( )

Definition at line 571 of file condformats_serialization_generate.py.

References generate().

572 def main():
573  parser = argparse.ArgumentParser(description='CMS Condition DB Serialization generator.')
574  parser.add_argument('--verbose', '-v', action='count', help='Verbosity level. -v reports debugging information.', default=0)
575  parser.add_argument('--output' , '-o', action='store', help='Specifies the path to the output file written. Default: src/Serialization.cc')
576  parser.add_argument('--package', '-p', action='store', help='Specifies the path to the package to be processed. Default: the actual package')
577 
578  opts, args = parser.parse_known_args()
579 
580  logLevel = logging.INFO
581  if opts.verbose < 1 and opts.output and opts.package: # assume we're called by scram and reduce logging - but only if no verbose is requested
582  logLevel = logging.WARNING
583 
584  if opts.verbose >= 1:
585  logLevel = logging.DEBUG
586 
587  logging.basicConfig(
588  format = '[%(asctime)s] %(levelname)s: %(message)s',
589  level = logLevel,
590  )
591 
592  if opts.package: # we got a directory name to process, assume it's from scram and remove the last ('/src') dir from the path
593  pkgDir = opts.package
594  if pkgDir.endswith('/src') :
595  pkgDir, srcDir = os.path.split( opts.package )
596  os.chdir( pkgDir )
597  logging.info("Processing package in %s " % pkgDir)
598 
599  if opts.output:
600  logging.info("Writing serialization code to %s " % opts.output)
601 
602  SerializationCodeGenerator( scramFlags=args[1:] ).generate( opts.output )
void generate(uint32_t const nbins, float const *initValues, std::vector< float > &values)
def condformats_serialization_generate.sanitise (   var)

Definition at line 416 of file condformats_serialization_generate.py.

Referenced by condformats_serialization_generate.SerializationCodeGenerator.generate().

417 def sanitise(var):
418  return re.sub('[^a-zA-Z0-9.,-:]', '-', var)
419 
def condformats_serialization_generate.split_path (   path)

Definition at line 325 of file condformats_serialization_generate.py.

Referenced by init_filter().

326 def split_path(path):
327  folders = []
328 
329  while True:
330  path, folder = os.path.split(path)
331 
332  if folder != '':
333  folders.append(folder)
334  else:
335  if path != '':
336  folders.append(path)
337  break
338 
339  folders.reverse()
340 
341  return folders
342 

Variable Documentation

string condformats_serialization_generate.__author__ = 'Miguel Ojeda'

Definition at line 17 of file condformats_serialization_generate.py.

string condformats_serialization_generate.__copyright__ = 'Copyright 2014, CERN'

Definition at line 18 of file condformats_serialization_generate.py.

list condformats_serialization_generate.__credits__ = ['Giacomo Govi', 'Miguel Ojeda', 'Andreas Pfeiffer']

Definition at line 19 of file condformats_serialization_generate.py.

string condformats_serialization_generate.__email__ = 'mojedasa@cern.ch'

Definition at line 22 of file condformats_serialization_generate.py.

string condformats_serialization_generate.__license__ = 'Unknown'

Definition at line 20 of file condformats_serialization_generate.py.

string condformats_serialization_generate.__maintainer__ = 'Miguel Ojeda'

Definition at line 21 of file condformats_serialization_generate.py.

condformats_serialization_generate.clang_version = None

Definition at line 33 of file condformats_serialization_generate.py.

string condformats_serialization_generate.headers_template
Initial value:
1 = '''
2 #include "{headers}"
3 
4 #include <boost/serialization/base_object.hpp>
5 #include <boost/serialization/nvp.hpp>
6 #include <boost/serialization/export.hpp>
7 
8 #include "CondFormats/Serialization/interface/Equal.h"
9 #include "CondFormats/Serialization/interface/Instantiate.h"
10 
11 '''

Definition at line 35 of file condformats_serialization_generate.py.

string condformats_serialization_generate.instantiation_template
Initial value:
1 = '''COND_SERIALIZATION_INSTANTIATE({klass});
2 '''

Definition at line 58 of file condformats_serialization_generate.py.

string condformats_serialization_generate.serialize_method_base_object_template = ' ar & boost::serialization::make_nvp("{base_object_name_sanitised}", boost::serialization::base_object<{base_object_name}>(*this));'

Definition at line 51 of file condformats_serialization_generate.py.

string condformats_serialization_generate.serialize_method_begin_template
Initial value:
1 = '''template <class Archive>
2 void {klass}::serialize(Archive & ar, const unsigned int)
3 {{'''

Definition at line 47 of file condformats_serialization_generate.py.

string condformats_serialization_generate.serialize_method_end
Initial value:
1 = '''}
2 '''

Definition at line 55 of file condformats_serialization_generate.py.

string condformats_serialization_generate.serialize_method_member_template = ''' ar & boost::serialization::make_nvp("{member_name_sanitised}", {member_name});'''

Definition at line 53 of file condformats_serialization_generate.py.

tuple condformats_serialization_generate.skip_namespaces
Initial value:
1 = frozenset([
2  # Do not go inside anonymous namespaces (static)
3  '',
4 
5  # Do not go inside some standard namespaces
6  'std', 'boost', 'mpl_', 'boost_swap_impl',
7 
8  # Do not go inside some big namespaces coming from externals
9  'ROOT', 'edm', 'ora', 'coral', 'CLHEP', 'Geom', 'HepGeom',
10 ])

Definition at line 62 of file condformats_serialization_generate.py.