CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Functions | Variables
SiStripDAQPopCon Namespace Reference

Functions

def main
 
def runjob
 

Variables

string cfg_template = 'CondTools/SiStrip/python/SiStripO2O_cfg_template.py'
 
string jobDirVar = 'JOBDIR'
 

Function Documentation

def SiStripDAQPopCon.main ( )

Definition at line 122 of file SiStripDAQPopCon.py.

References runjob(), and submitPVValidationJobs.split().

123 def main():
124  parser = argparse.ArgumentParser(description='Run a single O2O job for SiStrip DAQ and upload the payloads to condition database.')
125  parser.add_argument('analyzer', metavar='ANALYZER', help='Which EDAnalyzer to use to create the payload.')
126  parser.add_argument('since', metavar='SINCE', type=str, help='Run number.')
127  parser.add_argument('cfgfile', metavar='CFGLINES', help='File containing configuration lines.')
128  parser.add_argument('--destTags', required=True, help='Destination tag name(s) for upload. Use comma to separate multiple values.')
129  parser.add_argument('--destDb', required=True, help='Destination DB to upload.')
130  parser.add_argument('--inputTag', required=True, help='Tag name to be used in the sqlite file.')
131  parser.add_argument('--condDbRead', default='oracle://cms_orcon_prod/CMS_CONDITIONS', help='Connection string for the DB from which the fast O2O retrives payloads.')
132  parser.add_argument('--hashmapDb', default='', help='DB to read and write config-to-payload hash (for fast O2O).')
133  parser.add_argument('--skiplistFile', default='', help='File containing the devices to be skipped in G1 O2O.')
134  parser.add_argument('--whitelistFile', default='', help='File of the whitelisted devices in G1 O2O.')
135 
136  parser.add_argument('--no-upload', action="store_true", default=False, help='Do not upload payload. Default: %(default)s.')
137  parser.add_argument('--use-uploader', action="store_true", default=False, help='Use conditionUploader instead of conddb copy. Default: %(default)s.')
138  parser.add_argument('--bookkeeping-db', default='prod', choices=['prod', 'dev', 'private'], help='Bookkeeping database for fast O2O and G1 O2O. Default: %(default)s.')
139  parser.add_argument('--debug', action="store_true", default=False, help='Switch on debug mode. Default: %(default)s.')
140  args = parser.parse_args()
141 
142  loglevel = logging.INFO
143  if args.debug:
144  loglevel = logging.DEBUG
145  if args.bookkeeping_db == 'prod':
146  args.bookkeeping_db = 'dev'
147  logging.basicConfig(level=loglevel, format='[%(asctime)s] %(levelname)s: %(message)s')
148 
149  if not args.since.isdigit():
150  raise RuntimeError('Since (=%s) must be a valid run number!'%(args.since))
151 
152  args.destTags = args.destTags.strip().split(',')
153 
154  logging.info('Running O2O %s on machine [%s]' % (args.analyzer, socket.gethostname()))
155 
156  try:
157  jobdirbase = os.environ[jobDirVar]
158  except KeyError:
159  jobdirbase = '/tmp'
160  logging.warning('%s not set in env, will use %s' % (jobDirVar, jobdirbase))
161 
162  # change filepaths in args to abs path
163  args.cfgfile = os.path.abspath(args.cfgfile)
164  if args.skiplistFile:
165  args.skiplistFile = os.path.abspath(args.skiplistFile)
166  if args.whitelistFile:
167  args.whitelistFile = os.path.abspath(args.whitelistFile)
168 
169  # change to O2O working directory
170  jobdir = os.path.join(jobdirbase, args.since, args.analyzer)
171  if not os.path.exists(jobdir):
172  os.makedirs(jobdir)
173  os.chdir(jobdir)
174  logging.info('Running O2O in %s' % jobdir)
175 
176  # run job and upload
177  runjob(args)
178 
179  logging.info('Done!')
180 
181 
def SiStripDAQPopCon.runjob (   args)

Definition at line 20 of file SiStripDAQPopCon.py.

References validate-o2o-wbm.f, submitPVValidationJobs.split(), and str.

Referenced by main().

20 
21 def runjob(args):
22  if args.debug:
23  logging.debug(str(args))
24 
25  # read cfglines from input cfgfile
26  with open(args.cfgfile) as cfgfile:
27  cfglines = cfgfile.read()
28  logging.debug(cfglines)
29 
30  # create config from template
31  job_file = 'cfg_{type}_{run}.py'.format(type=args.analyzer, run=args.since)
32  output_db = '{type}_{run}.db'.format(type=args.analyzer, run=args.since)
33  if os.path.exists(output_db):
34  logging.info('Output sqlite file %s already exists! Deleting...' % output_db)
35  os.remove(output_db)
36  hashmap_db = 'hashmap_{type}_{run}.db'.format(type=args.analyzer, run=args.since)
37  if os.path.exists(hashmap_db):
38  logging.info('Hashmap sqlite file %s already exists! Deleting...' % hashmap_db)
39  os.remove(hashmap_db)
40  replace_dict = {'_CFGLINES_' : cfglines.replace('\\', ''),
41  '_ANALYZER_' : args.analyzer,
42  '_USEANALYSIS_':'False',
43  '_CONDDB_' : args.condDbRead,
44  '_DBFILE_' : 'sqlite:///%s' % output_db,
45  '_TARGETTAG_': args.inputTag,
46  '_RUNNUMBER_': args.since,
47  '_HASHMAPDB_': args.hashmapDb,
48  '_MAPDBFILE_': 'sqlite:///%s' % hashmap_db,
49  '_SKIPPED_' : '',
50  '_WHITELISTED_': '',
51  }
52  if args.analyzer == 'SiStripO2OApvGain':
53  # special treatment for G1 O2O
54  skipped = ''
55  if args.skiplistFile:
56  with open(args.skiplistFile) as skipfile:
57  skipped = skipfile.read()
58  else:
59  logging.warning('Skipped module list not provided! No module will be skipped...')
60  whitelisted = ''
61  if args.whitelistFile:
62  with open(args.whitelistFile) as wfile:
63  whitelisted = wfile.read()
64  else:
65  logging.warning('Module whitelist not provided!')
66  replace_dict['_USEANALYSIS_'] = 'True'
67  replace_dict['_SKIPPED_'] = skipped
68  replace_dict['_WHITELISTED_'] = whitelisted
69  replace_dict['_HASHMAPDB_'] = ''
70  replace_dict['_MAPDBFILE_'] = ''
71 
72  # find template cfg file
73  for basedir in os.environ['CMSSW_SEARCH_PATH'].split(':'):
74  templatefile = os.path.join(basedir, cfg_template)
75  if os.path.exists(templatefile):
76  logging.info('Use template config file %s' % templatefile)
77  break
78  config = helper.insert_to_file(templatefile, job_file, replace_dict)
79  logging.info('Start running O2O...')
80  logging.debug(' ... config:\n%s\n' % config)
81 
82  # run cmssw job: raise error if failed
83  command = 'cmsRun %s' % job_file
84  pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
85  atexit.register(partial(helper.kill_subproc_noexcept, pipe))
86  out = pipe.communicate()[0]
87  if isinstance(out, bytes):
88  out = out.decode("utf8", "replace")
89  logging.info('\n%s\n' % out)
90  logging.info('@@@CMSSW job return code = %d@@@' % pipe.returncode)
91  if pipe.returncode != 0:
92  raise RuntimeError('O2O job FAILED!')
93 
94  # upload: raise error if failed
95  if args.no_upload:
96  logging.info('Will not run uploading as requested!')
97  return
98  if args.use_uploader:
99  f = helper.upload_payload
100  else:
101  f = helper.copy_payload
102  f(dbFile=output_db, inputTag=args.inputTag, destTags=args.destTags, destDb=args.destDb, since=args.since,
103  userText='{type}, run: {run}'.format(type=args.analyzer, run=args.since))
104 
105  # post O2O tasks: bookkeeping for fast O2O or G1 O2O
106  if args.analyzer == 'SiStripO2OApvGain':
107  logging.info('Writting bookkeeping info to database.')
108  dbmgr = DbManagerGain(args.bookkeeping_db)
109  dbmgr.update_gain_logs(args.since, job_file)
110  else:
111  logging.info('Updating config-to-payload hash map to database.')
112  dbmgr = DbManagerDAQ(args.bookkeeping_db)
113  dbmgr.update_hashmap(hashmap_db)
114 
115  # clean up
116  try:
117  os.remove(output_db)
118  os.remove(output_db.replace('.db', '.txt'))
119  os.remove(hashmap_db) # may not exist
120  except OSError:
121  pass
#define str(s)

Variable Documentation

string SiStripDAQPopCon.cfg_template = 'CondTools/SiStrip/python/SiStripO2O_cfg_template.py'

Definition at line 18 of file SiStripDAQPopCon.py.

string SiStripDAQPopCon.jobDirVar = 'JOBDIR'

Definition at line 17 of file SiStripDAQPopCon.py.