CMS 3D CMS Logo

o2oRun_SiStripDAQ.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 '''
3 Main Script to run all SiStrip DAQ O2Os at the same time.
4 @author: Huilin Qu
5 '''
6 
7 import os
8 import sys
9 import atexit
10 import logging
11 import argparse
12 import subprocess
13 import traceback
14 import json
15 from functools import partial
16 
17 import CondTools.SiStrip.o2o_helper as helper
18 
19 logDirVar = 'O2O_LOG_FOLDER'
20 
21 def run(args):
22  logging.debug(args)
23 
24  is_ok = True
25  status = {}
26  processes = {}
27 
28  for analyzer in args.analyzers:
29  o2ocmd = 'SiStripDAQPopCon.py {analyzer} {since} {cfgfile}'.format(
30  analyzer=analyzer, since=args.since, cfgfile=args.cfgfile)
31  o2ocmd += ' --destTags {destTags}'
32  o2ocmd += ' --destDb {destDb}'
33  o2ocmd += ' --inputTag {inputTag}'
34  o2ocmd += ' --condDbRead {condDbRead}'
35  o2ocmd += ' --hashmapDb {hashmapDb}'
36  if args.skiplistFile:
37  o2ocmd += ' --skiplistFile %s' % args.skiplistFile
38  if args.whitelistFile:
39  o2ocmd += ' --whitelistFile %s' % args.whitelistFile
40  if args.debug:
41  o2ocmd += ' --debug'
42 
43  jobname = analyzer.replace('O2O', '')
44  cmd = 'o2o --db {db} -v run -n {jobname} "{o2ocmd}"'.format(db=args.db, jobname=jobname, o2ocmd=o2ocmd)
45  logging.info('Start running command:\n %s' % cmd)
46 
47  processes[jobname] = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
48  atexit.register(partial(helper.kill_subproc_noexcept, processes[jobname]))
49 
50  for jobname in processes:
51  status[jobname] = {'job':None, 'upload':None, 'fast':None, 'changed':None}
52  p = processes[jobname]
53  log = p.communicate()[0]
54  logging.debug('=== log from o2o run ===\n%s' % log)
55  if p.returncode == 0:
56  logging.info('Job for %s finished successfully!' % jobname)
57  status[jobname]['job'] = True
58  status[jobname]['upload'] = True
59  for line in log.split('\n'):
60  if '@@@' not in line:
61  continue
62  if 'FastO2O' in line:
63  status[jobname]['fast'] = ('true' in line)
64  if 'PayloadChange' in line:
65  status[jobname]['changed'] = ('true' in line)
66  else:
67  logging.error('Job %s FAILED!' % jobname)
68  status[jobname]['job'] = '@@@CMSSW job return code = 0@@@' in log
69  status[jobname]['upload'] = '@@@Upload return code = 0@@@' in log
70  is_ok = False
71 
72  return is_ok, status
73 
74 def summary(args, is_ok, status, logfile):
75  summary = json.dumps(status, sort_keys=True, indent=2)
76  if is_ok:
77  logging.info('O2O finished successfully! Summary: %s' % summary)
78  else:
79  logging.error('O2O FAILED! Summary: %s' % summary)
80 
81  debugLabel = '[TEST] ' if args.debug else ''
82 
83  # send the summary email
84  helper.send_mail(subject='%sNew O2O, IOV: %s' % (debugLabel, args.since),
85  message=summary,
86  send_to=args.mail_to,
87  send_from=args.mail_from)
88  # send the detailed log
89  with open(logfile, 'rb') as log:
90  helper.send_mail(subject='%sNew O2O Log, IOV: %s' % (debugLabel, args.since),
91  message=log.read(),
92  send_to=args.mail_log_to,
93  send_from=args.mail_from)
94 
95 
96 def main():
97  parser = argparse.ArgumentParser(description='Run all SiStrip DAQ O2Os at the same time.')
98  parser.add_argument('since', metavar='SINCE', type=str, help='Run number.')
99  parser.add_argument('cfgfile', metavar='CFGLINES', help='File containing configuration lines.')
100  parser.add_argument('--skiplistFile', default='', help='File containing the devices to be skipped in G1 O2O.')
101  parser.add_argument('--whitelistFile', default='', help='File of the whitelisted devices in G1 O2O.')
102 
103  parser.add_argument('--analyzers',
104  default='SiStripO2OBadStrip,SiStripO2OFedCabling,SiStripO2OLatency,SiStripO2ONoises,SiStripO2OPedestals,SiStripO2OThreshold',
105  help='Which EDAnalyzers to run.')
106  parser.add_argument('--mail-from', default='trk.o2o@cern.ch', help='Account to send email notification.')
107  parser.add_argument('--mail-to', default='cms-tracker-o2o-notification@cern.ch', help='List of O2O notification recipients.')
108  parser.add_argument('--mail-log-to', default='trk.o2o@cern.ch', help='List of O2O log recipients.')
109  parser.add_argument('--db', default='pro', help='The database for o2o job management: pro ( for prod ) or dev ( for prep ). Default: %(default)s.')
110  parser.add_argument('--debug', action="store_true", default=False, help='Switch on debug mode. Default: %(default)s.')
111 
112  args = parser.parse_args()
113  if args.debug:
114  args.mail_to = args.mail_log_to
115 
116  args.analyzers = args.analyzers.strip().split(',')
117  args.mail_to = args.mail_to.strip().split(',')
118  args.mail_log_to = args.mail_log_to.strip().split(',')
119 
120  # Should NOT use logging before it's set up
121  try:
122  logdir = os.environ[logDirVar] if logDirVar in os.environ else '/tmp'
123  if not os.path.exists(logdir):
124  os.makedirs(logdir)
125  logfile = os.path.join(logdir, 'SiStripsO2O_Run%s.log' % str(args.since))
126  loglevel = logging.DEBUG if args.debug else logging.INFO
127  helper.configLogger(logfile, loglevel)
128  except Exception:
129  # in case we failed before logging is set up
130  # print the error, send an email, and exit
131  helper.send_mail('O2O Failure, IOV: %s' % args.since, traceback.format_exc(), args.mail_to, args.mail_from)
132  raise
133 
134  try:
135  is_ok, status = run(args)
136  summary(args, is_ok, status, logfile)
137  except Exception:
138  # in case we failed before logging is set up
139  # print the error, send an email, and exit
140  helper.send_mail('O2O Failure, IOV: %s' % args.since, traceback.format_exc(), args.mail_to, args.mail_from)
141  raise
142 
143  if not is_ok:
144  return ' --- O2O FAILED! ---'
145 
146 if __name__ == '__main__':
147  sys.exit(main())
def summary(args, is_ok, status, logfile)
Definition: main.py:1
#define str(s)
double split
Definition: MVATrainer.cc:139