3 Main Script to run all SiStrip DAQ O2Os at the same time. 15 from functools
import partial
17 import CondTools.SiStrip.o2o_helper
as helper
19 logDirVar =
'O2O_LOG_FOLDER' 28 for analyzer
in args.analyzers:
29 o2ocmd =
'SiStripDAQPopCon.py {analyzer} {since} {cfgfile}'.
format(
30 analyzer=analyzer, since=args.since, cfgfile=args.cfgfile)
31 o2ocmd +=
' --destTags {destTags}' 32 o2ocmd +=
' --destDb {destDb}' 33 o2ocmd +=
' --inputTag {inputTag}' 34 o2ocmd +=
' --condDbRead {condDbRead}' 35 o2ocmd +=
' --hashmapDb {hashmapDb}' 37 o2ocmd +=
' --skiplistFile %s' % args.skiplistFile
38 if args.whitelistFile:
39 o2ocmd +=
' --whitelistFile %s' % args.whitelistFile
43 jobname = analyzer.replace(
'O2O',
'')
44 cmd =
'o2o --db {db} -v run -n {jobname} "{o2ocmd}"'.
format(db=args.db, jobname=jobname, o2ocmd=o2ocmd)
45 logging.info(
'Start running command:\n %s' % cmd)
47 processes[jobname] = subprocess.Popen(cmd, shell=
True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
48 atexit.register(partial(helper.kill_subproc_noexcept, processes[jobname]))
50 for jobname
in processes:
51 status[jobname] = {
'job':
None,
'upload':
None,
'fast':
None,
'changed':
None}
52 p = processes[jobname]
53 log = p.communicate()[0]
54 logging.debug(
'=== log from o2o run ===\n%s' % log)
56 logging.info(
'Job for %s finished successfully!' % jobname)
57 status[jobname][
'job'] =
True 58 status[jobname][
'upload'] =
True 59 for line
in log.split(
'\n'):
63 status[jobname][
'fast'] = (
'true' in line)
64 if 'PayloadChange' in line:
65 status[jobname][
'changed'] = (
'true' in line)
67 logging.error(
'Job %s FAILED!' % jobname)
68 status[jobname][
'job'] =
'@@@CMSSW job return code = 0@@@' in log
69 status[jobname][
'upload'] =
'@@@Upload return code = 0@@@' in log
74 def summary(args, is_ok, status, logfile):
75 summary = json.dumps(status, sort_keys=
True, indent=2)
77 logging.info(
'O2O finished successfully! Summary: %s' % summary)
79 logging.error(
'O2O FAILED! Summary: %s' % summary)
81 debugLabel =
'[TEST] ' if args.debug
else '' 84 helper.send_mail(subject=
'%sNew O2O, IOV: %s' % (debugLabel, args.since),
87 send_from=args.mail_from)
89 with open(logfile,
'rb')
as log:
90 helper.send_mail(subject=
'%sNew O2O Log, IOV: %s' % (debugLabel, args.since),
92 send_to=args.mail_log_to,
93 send_from=args.mail_from)
97 parser = argparse.ArgumentParser(description=
'Run all SiStrip DAQ O2Os at the same time.')
98 parser.add_argument(
'since', metavar=
'SINCE', type=str, help=
'Run number.')
99 parser.add_argument(
'cfgfile', metavar=
'CFGLINES', help=
'File containing configuration lines.')
100 parser.add_argument(
'--skiplistFile', default=
'', help=
'File containing the devices to be skipped in G1 O2O.')
101 parser.add_argument(
'--whitelistFile', default=
'', help=
'File of the whitelisted devices in G1 O2O.')
103 parser.add_argument(
'--analyzers',
104 default=
'SiStripO2OBadStrip,SiStripO2OFedCabling,SiStripO2OLatency,SiStripO2ONoises,SiStripO2OPedestals,SiStripO2OThreshold',
105 help=
'Which EDAnalyzers to run.')
106 parser.add_argument(
'--mail-from', default=
'trk.o2o@cern.ch', help=
'Account to send email notification.')
107 parser.add_argument(
'--mail-to', default=
'cms-tracker-o2o-notification@cern.ch', help=
'List of O2O notification recipients.')
108 parser.add_argument(
'--mail-log-to', default=
'trk.o2o@cern.ch', help=
'List of O2O log recipients.')
109 parser.add_argument(
'--db', default=
'pro', help=
'The database for o2o job management: pro ( for prod ) or dev ( for prep ). Default: %(default)s.')
110 parser.add_argument(
'--debug', action=
"store_true", default=
False, help=
'Switch on debug mode. Default: %(default)s.')
112 args = parser.parse_args()
114 args.mail_to = args.mail_log_to
116 args.analyzers = args.analyzers.strip().
split(
',')
117 args.mail_to = args.mail_to.strip().
split(
',')
118 args.mail_log_to = args.mail_log_to.strip().
split(
',')
122 logdir = os.environ[logDirVar]
if logDirVar
in os.environ
else '/tmp' 123 if not os.path.exists(logdir):
125 logfile = os.path.join(logdir,
'SiStripsO2O_Run%s.log' %
str(args.since))
126 loglevel = logging.DEBUG
if args.debug
else logging.INFO
127 helper.configLogger(logfile, loglevel)
131 helper.send_mail(
'O2O Failure, IOV: %s' % args.since, traceback.format_exc(), args.mail_to, args.mail_from)
135 is_ok, status =
run(args)
136 summary(args, is_ok, status, logfile)
140 helper.send_mail(
'O2O Failure, IOV: %s' % args.since, traceback.format_exc(), args.mail_to, args.mail_from)
144 return ' --- O2O FAILED! ---' 146 if __name__ ==
'__main__':
def summary(args, is_ok, status, logfile)