CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Functions | Variables
cmsswFiletrace Namespace Reference

Functions

def addprefixinfo
 
def cleanupenv
 
def formatfile
 
def help
 
def main
 
def searchinpath
 
def setupenv
 
def trace_command
 
def trace_python
 
def writeoutput
 

Variables

string ARGV0 = ""
 
 FLAT_OUTPUT = False
 
list IGNORE_DIRS
 
string OUTFILE_FILES = "callfiles"
 
string OUTFILE_TREE = "calltree"
 
list PREFIXINFO = []
 
list STRIPPATHS
 
list WRAP_SCRIPTS = ["cmsDriver.py" ]
 

Function Documentation

def cmsswFiletrace.addprefixinfo (   argv)

Definition at line 31 of file cmsswFiletrace.py.

Referenced by main().

31 
32 def addprefixinfo(argv):
33  cwd = os.path.abspath(os.getcwd())
34  wf = re.match(".*/(\d+\.\d+)_", cwd)
35  if wf:
36  PREFIXINFO.append("wf")
37  PREFIXINFO.append(wf.groups()[0])
38  online = re.match("(.*/)?(.*)_dqm_sourceclient-live_cfg\.py", argv[0])
39  if online:
40  PREFIXINFO.append("online")
41  PREFIXINFO.append(online.groups()[1])
42  step = re.match("(step\d+)_.*\.py", argv[0])
43  if step:
44  PREFIXINFO.append(step.groups()[0])
45  processing = re.match("step\d+_.*(RECO|ALCA|HARVEST).*\.py", argv[0])
46  if processing:
47  PREFIXINFO.append(processing.groups()[0])
48  if not PREFIXINFO:
49  PREFIXINFO.append(argv[0])
def cmsswFiletrace.cleanupenv (   tmpdir)

Definition at line 66 of file cmsswFiletrace.py.

References filterCSVwithJSON.copy, and print().

Referenced by trace_command().

66 
67 def cleanupenv(tmpdir):
68  #with open(os.environ["CMSSWCALLTREE"], "a") as f:
69  # print("}", file=f)
70  print("+Cleaning up ", tmpdir)
71  copy(os.environ["CMSSWCALLTREE"], ".")
72  copy(os.environ["CMSSWCALLFILES"], ".")
73  rmtree(tmpdir)
74 
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def cmsswFiletrace.formatfile (   filename)

Definition at line 85 of file cmsswFiletrace.py.

Referenced by writeoutput(), and cmsswConfigtrace.writeoutput().

85 
86 def formatfile(filename):
87  filename = os.path.abspath(filename)
88  for pfx in STRIPPATHS:
89  if filename.startswith(pfx):
90  filename = filename[len(pfx):]
91  return filename
def cmsswFiletrace.help ( )

Definition at line 228 of file cmsswFiletrace.py.

References join(), and print().

Referenced by main().

228 
229 def help():
230  print("Usage: %s <some cmssw commandline>" % (sys.argv[0]))
231  print(" The given programs will be executed, instrumenting calls to %s and cmsRun." % (", ".join(WRAP_SCRIPTS)))
232  print(" cmsRun will not actually run cmssw, but all the Python code will be executed and instrumentd. The results are written to the files `%s` and `%s` in the same directory." % (OUTFILE_FILES, OUTFILE_TREE))
233  if FLAT_OUTPUT:
234  print(" The callgraph output file can be processed with Brendan Gregg's FlameGraph tool.")
235  else:
236  print(" The callgraph output lists edges pointing from each function to the one calling it.")
237 
238  print("Examples:")
239  print(" %s runTheMatrix.py -l 1000 --ibeos" % sys.argv[0])
240  print( "%s cmsRun rpc_dqm_sourceclient-live_cfg.py" % sys.argv[0])
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def cmsswFiletrace.main ( )

Definition at line 241 of file cmsswFiletrace.py.

References addprefixinfo(), alcazmumu_cfi.filter, help(), print(), submitPVValidationJobs.split(), trace_command(), and trace_python().

241 
242 def main():
243  print("+Running cmsswfiletrace...")
244  global ARGV0
245  ARGV0 = sys.argv[0]
246  for s in WRAP_SCRIPTS:
247  if sys.argv[0].endswith(s):
248  print("+Wrapping %s..." % s)
249  addprefixinfo(sys.argv)
250  tmppath = os.path.dirname(sys.argv[0])
251  path = filter(
252  lambda s: not s.startswith(tmppath),
253  os.environ["PATH"].split(":")
254  )
255  STRIPPATHS.append(os.environ["CMSSWCALLBASE"])
256  trace_python([s] + sys.argv[1:], path)
257  return
258  if sys.argv[0].endswith('cmsRun'):
259  print("+Wrapping cmsRun...")
260  addprefixinfo(sys.argv[1:])
261  STRIPPATHS.append(os.environ["CMSSWCALLBASE"])
262  trace_python(sys.argv[1:], ["."])
263  return
264  if len(sys.argv) <= 1:
265  help()
266  return
267  # else
268  print("+Running command with tracing %s..." % sys.argv[1:])
269  trace_command(sys.argv[1:])
270 
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def cmsswFiletrace.searchinpath (   progname,
  path 
)

Definition at line 92 of file cmsswFiletrace.py.

References print().

92 
93 def searchinpath(progname, path):
94  # Search $PATH. There seems to be no pre-made function for this.
95  for entry in path:
96  file_path = os.path.join(entry, progname)
97  if os.path.isfile(file_path):
98  break
99  if not os.path.isfile(file_path):
100  print("+Cannot find program (%s) in modified $PATH (%s)." % (progname, path))
101  sys.exit(1)
102  print("+Found %s as %s in %s." % (progname, file_path, path))
103  return file_path
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def cmsswFiletrace.setupenv ( )

Definition at line 50 of file cmsswFiletrace.py.

References print().

Referenced by trace_command().

50 
51 def setupenv():
52  bindir = tempfile.mkdtemp()
53  print("+Setting up in ", bindir)
54  for s in WRAP_SCRIPTS:
55  os.symlink(ARGV0, bindir + "/" + s)
56  os.symlink(ARGV0, bindir + "/cmsRun")
57  os.environ["PATH"] = bindir + ":" + os.environ["PATH"]
58  os.environ["CMSSWCALLTREE"] = bindir + "/" + OUTFILE_TREE
59  os.environ["CMSSWCALLFILES"] = bindir + "/" + OUTFILE_FILES
60  os.environ["CMSSWCALLBASE"] = os.path.abspath(os.getcwd()) + "/"
61  with open(os.environ["CMSSWCALLTREE"], "w") as f:
62  pass
63  with open(os.environ["CMSSWCALLFILES"], "w") as f:
64  pass
65  return bindir
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def cmsswFiletrace.trace_command (   argv)

Definition at line 75 of file cmsswFiletrace.py.

References cleanupenv(), and setupenv().

Referenced by main().

75 
76 def trace_command(argv):
77  tmpdir = None
78  if not "CMSSWCALLTREE" in os.environ:
79  tmpdir = setupenv()
80 
81  subprocess.call(argv)
82 
83  if tmpdir:
84  cleanupenv(tmpdir)
def cmsswFiletrace.trace_python (   prog_argv,
  path 
)

Definition at line 155 of file cmsswFiletrace.py.

Referenced by main().

155 
156 def trace_python(prog_argv, path):
157  files = set()
158  callgraph = defaultdict(lambda: set())
159 
160  def nop_trace(frame, why, arg):
161  pass
162 
163  def tracefunc(frame, why, arg):
164  if why == 'call':
165  code = frame.f_code
166  # compared to the `trace` module, we don't attempt to find class names here
167  filename = code.co_filename
168 
169  for d in IGNORE_DIRS:
170  if filename.startswith(d):
171  sys.settrace(nop_trace)
172  return wait_for_return
173 
174  funcname = code.co_name
175  code = frame.f_back.f_code
176  p_filename = code.co_filename
177  p_funcname = code.co_name
178 
179  files.add(filename)
180  callgraph[(filename, funcname)].add((p_filename, p_funcname))
181  return None
182 
183  def wait_for_return(frame, why, arg):
184  if why == 'return':
185  sys.settrace(tracefunc)
186  return wait_for_return
187 
188  sys.argv = prog_argv
189  progname = prog_argv[0]
190 
191 
192  file_path = searchinpath(progname, path)
193  try:
194  with open(file_path) as fp:
195  code = compile(fp.read(), progname, 'exec')
196  # try to emulate __main__ namespace as much as possible
197  globals = {
198  '__file__': progname,
199  '__name__': '__main__',
200  '__package__': None,
201  '__cached__': None,
202  }
203 
204  # would be too easy if this covered all the cases...
205  atexit.register(lambda: writeoutput(callgraph, files))
206  # cmsDriver calls cmsRun via exec (execvpe specifically), so we also need
207  # to hook that...
208  old_execvpe = os.execvpe
209  def exec_hook(*args):
210  writeoutput(callgraph, files)
211  old_execvpe(*args)
212  os.execvpe = exec_hook
213 
214  # now turn on the traceing
215  sys.settrace(tracefunc)
216  try:
217  exec code in globals, globals
218  finally:
219  sys.settrace(None)
220 
221  except OSError as err:
222  print("+Cannot run file %r because: %s" % (sys.argv[0], err))
223  sys.exit(1)
224  except SystemExit:
225  pass
226  # this is not necessarily reached at all.
227  sys.exit(0)
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
PVValHelper::add
void add(std::map< std::string, TH1 * > &h, TH1 *hist)
Definition: PVValidationHelpers.cc:12
def cmsswFiletrace.writeoutput (   callgraph,
  files 
)

Definition at line 104 of file cmsswFiletrace.py.

References diffTreeTool.format(), formatfile(), join(), GetRecoTauVFromDQM_MC_cff.next, and print().

104 
105 def writeoutput(callgraph, files):
106  progname = ", ".join(PREFIXINFO)
107  print("+Done running %s, writing output..." % progname)
108 
109  def format(func):
110  filename, funcname = func
111  return "%s::%s" % (formatfile(filename), funcname)
112 
113  def callpath(func):
114  # climb up in the call graph until we find a node without callers (this is
115  # the entry point, the traced call itself). There may be cycles, but any
116  # node is reachable from the entry point, so no backtracking required.
117  path = []
118  seen = set()
119  parents = {func}
120  timeout = 100 # go no more than this deep
121  while parents:
122  if len(parents) == 1:
123  func = next(iter(parents))
124  seen.add(func)
125  path.append(format(func))
126  if len(parents) > 1:
127  for func in parents:
128  if not func in seen:
129  break
130  if func in seen:
131  # somehow we got stuck in a loop and can't get out. So maybe
132  # backtracking is needed in some situations?
133  # Abort with a partial path for now.
134  return path
135  seen.add(func)
136  path.append(format(func) + "+")
137  parents = callgraph[func]
138  timeout -= 1
139  if timeout == 0:
140  print(seen, path, parents, func)
141  raise Exception('Call path too deep, aborting')
142  return path[:-1]
143 
144  with open(os.environ["CMSSWCALLFILES"], "a") as outfile:
145  for f in files:
146  print("%s: %s" % (progname, formatfile(f)), file=outfile)
147  with open(os.environ["CMSSWCALLTREE"], "a") as outfile:
148  if FLAT_OUTPUT:
149  for func in callgraph.keys():
150  print("%s: %s 1" % (progname, ";".join(reversed(callpath(func)))), file=outfile)
151  else:
152  for func in callgraph.keys():
153  for pfunc in callgraph[func]:
154  print("%s: %s -> %s" % (progname, format(func), format(pfunc)), file=outfile)
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
GetRecoTauVFromDQM_MC_cff.next
tuple next
Definition: GetRecoTauVFromDQM_MC_cff.py:31

Variable Documentation

string cmsswFiletrace.ARGV0 = ""

Definition at line 29 of file cmsswFiletrace.py.

cmsswFiletrace.FLAT_OUTPUT = False

Definition at line 18 of file cmsswFiletrace.py.

list cmsswFiletrace.IGNORE_DIRS
Initial value:
1 = [
2  os.path.dirname(os.__file__),
3  FWCore.ParameterSet.Types.__file__,
4 ]

Definition at line 21 of file cmsswFiletrace.py.

string cmsswFiletrace.OUTFILE_FILES = "callfiles"

Definition at line 17 of file cmsswFiletrace.py.

string cmsswFiletrace.OUTFILE_TREE = "calltree"

Definition at line 16 of file cmsswFiletrace.py.

list cmsswFiletrace.PREFIXINFO = []

Definition at line 28 of file cmsswFiletrace.py.

list cmsswFiletrace.STRIPPATHS
Initial value:
1 = [ # we will add the base dir from CMSSWCALLBASE env var here
2  os.environ["CMSSW_BASE"] + "/python/", os.environ["CMSSW_RELEASE_BASE"] + "/python/",
3  os.environ["CMSSW_BASE"] + "/cfipython/", os.environ["CMSSW_RELEASE_BASE"] + "/cfipython/"]

Definition at line 25 of file cmsswFiletrace.py.

list cmsswFiletrace.WRAP_SCRIPTS = ["cmsDriver.py" ]

Definition at line 20 of file cmsswFiletrace.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5