CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
Classes | Functions
condformats_serialization_generate Namespace Reference

Classes

class  SerializationCodeGenerator
 

Functions

def get_basic_type_string (node)
 
def get_clang_version ()
 
def get_default_gcc_search_paths (gcc='g++', language='c++')
 
def get_diagnostics (translation_unit)
 
def get_flags (product_name, flags)
 
def get_serializable_classes_members (node, all_template_types=None, namespace='', only_from_path=None)
 
def get_statement (node)
 
def get_type_string (node)
 
def is_definition_by_loc (node)
 
def is_friend_decl (memkind)
 
def is_serializable_class (node)
 
def is_serializable_class_manual (node)
 
def log_flags (name, flags)
 
def main ()
 
def sanitise (var)
 
def split_path (path)
 

Function Documentation

◆ get_basic_type_string()

def condformats_serialization_generate.get_basic_type_string (   node)

Definition at line 122 of file condformats_serialization_generate.py.

122 def get_basic_type_string(node):
123  typekinds = {
124  clang.cindex.TypeKind.BOOL: 'bool',
125  clang.cindex.TypeKind.INT: 'int',
126  clang.cindex.TypeKind.LONG: 'long',
127  clang.cindex.TypeKind.UINT: 'unsigned int',
128  clang.cindex.TypeKind.ULONG: 'unsigned long',
129  clang.cindex.TypeKind.FLOAT: 'float',
130  clang.cindex.TypeKind.DOUBLE: 'double',
131  }
132 
133  if node.type.kind not in typekinds:
134  raise Exception('Not a known basic type.')
135 
136  return typekinds[node.type.kind]
137 
138 

Referenced by get_serializable_classes_members(), and get_type_string().

◆ get_clang_version()

def condformats_serialization_generate.get_clang_version ( )
Extract clang version and set global clang_version and also return the same value.

Definition at line 348 of file condformats_serialization_generate.py.

348 def get_clang_version():
349  """Extract clang version and set global clang_version and also return the same value."""
350  global clang_version
351  if clang_version is not None:
352  return clang_version
353  command = "clang --version | grep 'clang version' | sed 's/clang version//'"
354  logging.debug("Running: {0}".format(command))
355  (clang_version_major, clang_version_minor, clang_version_patchlevel) = subprocess.check_output(command, shell=True).splitlines()[0].decode('ascii').strip().split(" ")[0].split('.', 3)
356  clang_version = (int(clang_version_major), int(clang_version_minor), int(clang_version_patchlevel))
357  logging.debug("Detected Clang version: {0}".format(clang_version))
358  return clang_version
359 

References edm.decode(), createfilelist.int, cms::dd.split(), and digitizers_cfi.strip.

Referenced by is_friend_decl().

◆ get_default_gcc_search_paths()

def condformats_serialization_generate.get_default_gcc_search_paths (   gcc = 'g++',
  language = 'c++' 
)

Definition at line 386 of file condformats_serialization_generate.py.

386 def get_default_gcc_search_paths(gcc = 'g++', language = 'c++'):
387  command = 'echo "" | %s -x%s -v -E - 2>&1' % (gcc, language)
388  logging.debug('Running: %s', command)
389 
390  paths = []
391  in_list = False
392  for line in [l.decode("ascii") for l in subprocess.check_output(command, shell=True).splitlines()]:
393  if in_list:
394  if line == 'End of search list.':
395  break
396 
397  path = os.path.normpath(line.strip())
398 
399  # Intrinsics not handled by clang
400  # Note that /lib/gcc is found in other paths if not normalized,
401  # so has to go after normpath()
402  if '/lib/gcc/' in path:
403  continue
404 
405  paths.append('-I%s' % path)
406 
407  else:
408  if line == '#include <...> search starts here:':
409  in_list = True
410 
411  if not in_list:
412  raise Exception('Default GCC search paths not found.')
413 
414  return paths
415 

◆ get_diagnostics()

def condformats_serialization_generate.get_diagnostics (   translation_unit)

Definition at line 376 of file condformats_serialization_generate.py.

376 def get_diagnostics(translation_unit):
377  return map(lambda diag: {
378  'severity' : diag.severity,
379  'location' : diag.location,
380  'spelling' : diag.spelling,
381  'ranges' : diag.ranges,
382  'fixits' : diag.fixits,
383  }, translation_unit.diagnostics)
384 
385 

References genParticles_cff.map.

◆ get_flags()

def condformats_serialization_generate.get_flags (   product_name,
  flags 
)

Definition at line 343 of file condformats_serialization_generate.py.

343 def get_flags(product_name, flags):
344  command = "scram b echo_%s_%s | tail -1 | cut -d '=' -f '2-' | xargs -n1" % (product_name, flags)
345  logging.debug('Running: %s', command)
346  return subprocess.check_output(command, shell=True).splitlines()
347 

◆ get_serializable_classes_members()

def condformats_serialization_generate.get_serializable_classes_members (   node,
  all_template_types = None,
  namespace = '',
  only_from_path = None 
)

Definition at line 147 of file condformats_serialization_generate.py.

147 def get_serializable_classes_members(node, all_template_types=None, namespace='', only_from_path=None):
148  if all_template_types is None:
149  all_template_types = []
150 
151  logging.debug('%s', (node.spelling, all_template_types, namespace))
152  results = {}
153  for child in node.get_children():
154  if child.kind == clang.cindex.CursorKind.NAMESPACE:
155  # If we are in the root namespace, let's skip some common, big
156  # namespaces to improve speed and avoid serializing those.
157  if namespace == '':
158  if child.spelling in skip_namespaces:
159  continue
160 
161  # This skips compiler-specific stuff as well (e.g. __gnucxx...)
162  if child.spelling.startswith('_'):
163  continue
164 
165  logging.debug('Going into namespace %s', child.spelling)
166 
167  results.update(get_serializable_classes_members(child, all_template_types, namespace + child.spelling + '::', only_from_path))
168  continue
169 
170  if child.kind in [clang.cindex.CursorKind.CLASS_DECL, clang.cindex.CursorKind.STRUCT_DECL, clang.cindex.CursorKind.CLASS_TEMPLATE] and is_definition_by_loc(child):
171  logging.debug('Found struct/class/template definition: %s', child.spelling if child.spelling else '<anonymous>')
172 
173  if only_from_path is not None \
174  and child.location.file is not None \
175  and not child.location.file.name.startswith(only_from_path):
176  logging.debug('Skipping since it is an external of this package: %s', child.spelling)
177  continue
178 
179  serializable = is_serializable_class(child)
180  if serializable:
181  if child.spelling == '':
182  raise Exception('It is not possible to serialize anonymous/unnamed structs/classes.')
183 
185  logging.info('Found manual serializable struct/class/template: %s', child.spelling)
186  continue
187 
188  logging.info('Found serializable struct/class/template: %s', child.spelling)
189 
190  template_types = []
191  base_objects = []
192  members = []
193  transients = []
194  after_serialize = False
195  after_serialize_count = 0
196  for member in child.get_children():
197  if after_serialize:
198  if after_serialize_count == 2:
199  after_serialize = False
200  else:
201  after_serialize_count = after_serialize_count + 1
202 
203  if not is_friend_decl(member.kind):
204  raise Exception('Expected unexposed declaration (friend) after serialize() but found something else: looks like the COND_SERIALIZABLE macro has been changed without updating the script.')
205 
206  if 'COND_SERIALIZABLE' not in get_statement(member):
207  raise Exception('Could not find COND_SERIALIZABLE in the statement of the expected unexposed declarations (friends) after serialize(). Please fix the script/macro.')
208 
209  logging.debug('Skipping expected unexposed declaration (friend) after serialize().')
210  continue
211 
212  # Template type parameters (e.g. <typename T>)
213  if member.kind == clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER:
214  logging.info(' Found template type parameter: %s', member.spelling)
215  template_types.append(('typename', member.spelling))
216 
217  # Template non-type parameters (e.g. <int N>)
218  elif member.kind == clang.cindex.CursorKind.TEMPLATE_NON_TYPE_PARAMETER:
219  type_string = get_type_string(member)
220  if not type_string:
221  type_string = get_basic_type_string(member)
222  logging.info(' Found template non-type parameter: %s %s', type_string, member.spelling)
223  template_types.append((type_string, member.spelling))
224 
225  # Base objects
226  elif member.kind == clang.cindex.CursorKind.CXX_BASE_SPECIFIER:
227  # FIXME: .displayname gives sometimes things like "class mybase"
228  base_object = member.displayname
229  prefix = 'class '
230  if base_object.startswith(prefix):
231  base_object = base_object[len(prefix):]
232  logging.info(' Found base object: %s', base_object)
233  base_objects.append(base_object)
234 
235  # Member variables
236  elif member.kind == clang.cindex.CursorKind.FIELD_DECL and is_definition_by_loc(member):
237  # While clang 3.3 does not ignore unrecognized attributes
238  # (see http://llvm.org/viewvc/llvm-project?revision=165082&view=revision )
239  # for some reason they do not appear in the bindings yet
240  # so we just do it ourselves.
241 
242  # FIXME: To simplify and avoid parsing C++ ourselves, our transient
243  # attribute applies to *all* the variables declared in the same statement.
244  if 'COND_TRANSIENT' not in get_statement(member):
245  logging.info(' Found member variable: %s', member.spelling)
246  members.append(member.spelling)
247  else:
248  if serializable:
249  logging.info(' Found transient member variable: %s', member.spelling)
250  transients.append(member.spelling)
251  else:
252  raise Exception('Transient %s found for non-serializable class %s', member.spelling, child.spelling)
253 
254  elif member.kind == clang.cindex.CursorKind.FUNCTION_TEMPLATE and member.spelling == 'serialize':
255  after_serialize = True
256  logging.debug('Found serialize() method, skipping next two children which must be unexposed declarations.')
257 
258  elif member.kind in frozenset([
259  # For safety, we list all known kinds that we need to skip
260  # and raise in unknown cases (this helps catching problems
261  # with undefined classes)
262  clang.cindex.CursorKind.CONSTRUCTOR,
263  clang.cindex.CursorKind.DESTRUCTOR,
264  clang.cindex.CursorKind.CXX_METHOD,
265  clang.cindex.CursorKind.CXX_ACCESS_SPEC_DECL,
266  clang.cindex.CursorKind.FUNCTION_TEMPLATE,
267  clang.cindex.CursorKind.TYPEDEF_DECL,
268  clang.cindex.CursorKind.CLASS_DECL,
269  clang.cindex.CursorKind.ENUM_DECL,
270  clang.cindex.CursorKind.VAR_DECL,
271  clang.cindex.CursorKind.STRUCT_DECL,
272  clang.cindex.CursorKind.UNION_DECL,
273  clang.cindex.CursorKind.CONVERSION_FUNCTION,
274  clang.cindex.CursorKind.TYPE_REF,
275  clang.cindex.CursorKind.DECL_REF_EXPR,
276  clang.cindex.CursorKind.CLASS_TEMPLATE,
277  clang.cindex.CursorKind.TYPE_ALIAS_DECL,
278  ]):
279  logging.debug('Skipping member: %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind)
280 
281  elif is_friend_decl(member.kind):
282  statement = get_statement(member)
283 
284  # Friends are unexposed but they are not data to serialize
285  if 'friend' in statement:
286  # If we know about them, skip the warning
287  if \
288  'friend class ' in statement or \
289  'friend struct ' in statement or \
290  'friend std::ostream& operator<<(' in statement or \
291  'friend std::istream& operator>>(' in statement:
292  logging.debug('Skipping known friend: %s', statement.splitlines()[0])
293  continue
294 
295  # Otherwise warn
296  logging.warning('Unexposed declaration that looks like a friend declaration -- please check: %s %s %s %s %s', member.displayname, member.spelling, member.kind, member.type.kind, statement)
297  continue
298 
299  raise Exception('Unexposed declaration. This probably means (at the time of writing) that an unknown class was found (may happen, for instance, when the compiler does not find the headers for std::vector, i.e. missing -I option): %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))
300 
301  else:
302  statement = get_statement(member)
303  raise Exception('Unknown kind. Please fix the script: %s %s %s %s %s' % (member.displayname, member.spelling, member.kind, member.type.kind, statement))
304 
305  if template_types:
306  template_use = '%s<%s>' % (child.spelling, ', '.join([template_type_name for (_, template_type_name) in template_types]))
307  else:
308  template_use = child.spelling
309 
310  new_namespace = namespace + template_use
311 
312  new_all_template_types = all_template_types + [template_types]
313 
314  results[new_namespace] = (child, serializable, new_all_template_types, base_objects, members, transients)
315 
316  results.update(get_serializable_classes_members(child, new_all_template_types, new_namespace + '::', only_from_path))
317 
318  for (klass, (node, serializable, all_template_types, base_objects, members, transients)) in results.items():
319  if serializable and len(members) == 0:
320  logging.info('No non-transient members found for serializable class %s', klass)
321 
322  return results
323 
324 

References get_basic_type_string(), get_statement(), get_type_string(), is_definition_by_loc(), is_friend_decl(), is_serializable_class(), is_serializable_class_manual(), and join().

◆ get_statement()

def condformats_serialization_generate.get_statement (   node)

Definition at line 105 of file condformats_serialization_generate.py.

105 def get_statement(node):
106  # For some cursor kinds, their location is empty (e.g. translation units
107  # and attributes); either because of a bug or because they do not have
108  # a meaningful 'start' -- however, the extent is always available
109  if node.extent.start.file is None:
110  return None
111 
112  filename = node.extent.start.file.name
113  start = node.extent.start.offset
114  end = node.extent.end.offset
115 
116  with open(filename, 'rb') as fd:
117  source = fd.read().decode('latin-1')
118 
119  return source[start:source.find(';', end)]
120 
121 

References edm.decode().

Referenced by get_serializable_classes_members().

◆ get_type_string()

def condformats_serialization_generate.get_type_string (   node)

Definition at line 139 of file condformats_serialization_generate.py.

139 def get_type_string(node):
140  spelling = node.type.get_declaration().spelling
141  if spelling is not None:
142  return spelling
143 
144  return get_basic_type_string(node)
145 
146 

References get_basic_type_string().

Referenced by get_serializable_classes_members().

◆ is_definition_by_loc()

def condformats_serialization_generate.is_definition_by_loc (   node)

Definition at line 73 of file condformats_serialization_generate.py.

73 def is_definition_by_loc(node):
74  if node.get_definition() is None:
75  return False
76  if node.location is None or node.get_definition().location is None:
77  return False
78  return node.location == node.get_definition().location
79 

Referenced by get_serializable_classes_members(), is_serializable_class(), and is_serializable_class_manual().

◆ is_friend_decl()

def condformats_serialization_generate.is_friend_decl (   memkind)
Check if declaration is a friend

Definition at line 360 of file condformats_serialization_generate.py.

360 def is_friend_decl(memkind):
361  """Check if declaration is a friend"""
362  clangv = get_clang_version()
363  if clangv >= (4, 0, 0):
364  return memkind == clang.cindex.CursorKind.FRIEND_DECL
365  else:
366  return memkind == clang.cindex.CursorKind.UNEXPOSED_DECL
367  return false
368 

References get_clang_version().

Referenced by get_serializable_classes_members().

◆ is_serializable_class()

def condformats_serialization_generate.is_serializable_class (   node)

Definition at line 80 of file condformats_serialization_generate.py.

80 def is_serializable_class(node):
81  for child in node.get_children():
82  if child.spelling != 'serialize' or child.kind != clang.cindex.CursorKind.FUNCTION_TEMPLATE or is_definition_by_loc(child):
83  continue
84 
85  if [(x.spelling, x.kind, is_definition_by_loc(x), x.type.kind) for x in child.get_children()] != [
86  ('Archive', clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER, True, clang.cindex.TypeKind.UNEXPOSED),
87  ('ar', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.LVALUEREFERENCE),
88  ('version', clang.cindex.CursorKind.PARM_DECL, True, clang.cindex.TypeKind.UINT),
89  ]:
90  continue
91 
92  return True
93 
94  return False
95 
96 

References is_definition_by_loc().

Referenced by get_serializable_classes_members().

◆ is_serializable_class_manual()

def condformats_serialization_generate.is_serializable_class_manual (   node)

Definition at line 97 of file condformats_serialization_generate.py.

98  for child in node.get_children():
99  if child.spelling == 'cond_serialization_manual' and child.kind == clang.cindex.CursorKind.CXX_METHOD and not is_definition_by_loc(child):
100  return True
101 
102  return False
103 
104 

References is_definition_by_loc().

Referenced by get_serializable_classes_members().

◆ log_flags()

def condformats_serialization_generate.log_flags (   name,
  flags 
)

Definition at line 369 of file condformats_serialization_generate.py.

369 def log_flags(name, flags):
370  logging.debug('%s = [', name)
371  for flag in flags:
372  logging.debug(' %s', flag)
373  logging.debug(']')
374 
375 

◆ main()

def condformats_serialization_generate.main ( )

Definition at line 571 of file condformats_serialization_generate.py.

571 def main():
572  parser = argparse.ArgumentParser(description='CMS Condition DB Serialization generator.')
573  parser.add_argument('--verbose', '-v', action='count', help='Verbosity level. -v reports debugging information.', default=0)
574  parser.add_argument('--output' , '-o', action='store', help='Specifies the path to the output file written. Default: src/Serialization.cc')
575  parser.add_argument('--package', '-p', action='store', help='Specifies the path to the package to be processed. Default: the actual package')
576 
577  opts, args = parser.parse_known_args()
578 
579  logLevel = logging.INFO
580  if opts.verbose < 1 and opts.output and opts.package: # assume we're called by scram and reduce logging - but only if no verbose is requested
581  logLevel = logging.WARNING
582 
583  if opts.verbose >= 1:
584  logLevel = logging.DEBUG
585 
586  logging.basicConfig(
587  format = '[%(asctime)s] %(levelname)s: %(message)s',
588  level = logLevel,
589  )
590 
591  if opts.package: # we got a directory name to process, assume it's from scram and remove the last ('/src') dir from the path
592  pkgDir = opts.package
593  if pkgDir.endswith('/src') :
594  pkgDir, srcDir = os.path.split( opts.package )
595  os.chdir( pkgDir )
596  logging.info("Processing package in %s " % pkgDir)
597 
598  if opts.output:
599  logging.info("Writing serialization code to %s " % opts.output)
600 
601  SerializationCodeGenerator( scramFlags=args[1:] ).generate( opts.output )
602 

References models.generate().

◆ sanitise()

def condformats_serialization_generate.sanitise (   var)

Definition at line 416 of file condformats_serialization_generate.py.

416 def sanitise(var):
417  return re.sub('[^a-zA-Z0-9.,-:]', '-', var)
418 
419 

Referenced by condformats_serialization_generate.SerializationCodeGenerator.generate().

◆ split_path()

def condformats_serialization_generate.split_path (   path)

Definition at line 325 of file condformats_serialization_generate.py.

325 def split_path(path):
326  folders = []
327 
328  while True:
329  path, folder = os.path.split(path)
330 
331  if folder != '':
332  folders.append(folder)
333  else:
334  if path != '':
335  folders.append(path)
336  break
337 
338  folders.reverse()
339 
340  return folders
341 
342 
condformats_serialization_generate.get_default_gcc_search_paths
def get_default_gcc_search_paths(gcc='g++', language='c++')
Definition: condformats_serialization_generate.py:386
condformats_serialization_generate.get_clang_version
def get_clang_version()
Definition: condformats_serialization_generate.py:348
condformats_serialization_generate.get_type_string
def get_type_string(node)
Definition: condformats_serialization_generate.py:139
condformats_serialization_generate.split_path
def split_path(path)
Definition: condformats_serialization_generate.py:325
condformats_serialization_generate.get_basic_type_string
def get_basic_type_string(node)
Definition: condformats_serialization_generate.py:122
digitizers_cfi.strip
strip
Definition: digitizers_cfi.py:19
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:17
condformats_serialization_generate.is_definition_by_loc
def is_definition_by_loc(node)
Definition: condformats_serialization_generate.py:73
models.generate
def generate(map_blobs=False, class_name=None)
Definition: models.py:189
condformats_serialization_generate.sanitise
def sanitise(var)
Definition: condformats_serialization_generate.py:416
cms::dd::split
std::vector< std::string_view > split(std::string_view, const char *)
condformats_serialization_generate.is_serializable_class
def is_serializable_class(node)
Definition: condformats_serialization_generate.py:80
condformats_serialization_generate.main
def main()
Definition: condformats_serialization_generate.py:571
condformats_serialization_generate.log_flags
def log_flags(name, flags)
Definition: condformats_serialization_generate.py:369
condformats_serialization_generate.is_serializable_class_manual
def is_serializable_class_manual(node)
Definition: condformats_serialization_generate.py:97
Exception
createfilelist.int
int
Definition: createfilelist.py:10
condformats_serialization_generate.get_flags
def get_flags(product_name, flags)
Definition: condformats_serialization_generate.py:343
condformats_serialization_generate.get_serializable_classes_members
def get_serializable_classes_members(node, all_template_types=None, namespace='', only_from_path=None)
Definition: condformats_serialization_generate.py:147
edm::decode
bool decode(bool &, std::string const &)
Definition: types.cc:72
format
condformats_serialization_generate.get_statement
def get_statement(node)
Definition: condformats_serialization_generate.py:105
condformats_serialization_generate.get_diagnostics
def get_diagnostics(translation_unit)
Definition: condformats_serialization_generate.py:376
genParticles_cff.map
map
Definition: genParticles_cff.py:11
condformats_serialization_generate.is_friend_decl
def is_friend_decl(memkind)
Definition: condformats_serialization_generate.py:360