CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions
SiStripDAQPopCon Namespace Reference

Functions

def main ()
 
def runjob (args)
 

Detailed Description

Script that runs a single O2O for SiStrip DAQ.
@author: Huilin Qu

Function Documentation

def SiStripDAQPopCon.main ( )

Definition at line 120 of file SiStripDAQPopCon.py.

References runjob(), and split.

120 def main():
121  parser = argparse.ArgumentParser(description='Run a single O2O job for SiStrip DAQ and upload the payloads to condition database.')
122  parser.add_argument('analyzer', metavar='ANALYZER', help='Which EDAnalyzer to use to create the payload.')
123  parser.add_argument('since', metavar='SINCE', type=str, help='Run number.')
124  parser.add_argument('cfgfile', metavar='CFGLINES', help='File containing configuration lines.')
125  parser.add_argument('--destTags', required=True, help='Destination tag name(s) for upload. Use comma to separate multiple values.')
126  parser.add_argument('--destDb', required=True, help='Destination DB to upload.')
127  parser.add_argument('--inputTag', required=True, help='Tag name to be used in the sqlite file.')
128  parser.add_argument('--condDbRead', default='oracle://cms_orcon_prod/CMS_CONDITIONS', help='Connection string for the DB from which the fast O2O retrives payloads.')
129  parser.add_argument('--hashmapDb', default='', help='DB to read and write config-to-payload hash (for fast O2O).')
130  parser.add_argument('--skiplistFile', default='', help='File containing the devices to be skipped in G1 O2O.')
131  parser.add_argument('--whitelistFile', default='', help='File of the whitelisted devices in G1 O2O.')
132 
133  parser.add_argument('--no-upload', action="store_true", default=False, help='Do not upload payload. Default: %(default)s.')
134  parser.add_argument('--use-uploader', action="store_true", default=False, help='Use conditionUploader instead of conddb copy. Default: %(default)s.')
135  parser.add_argument('--bookkeeping-db', default='prod', choices=['prod', 'dev', 'private'], help='Bookkeeping database for fast O2O and G1 O2O. Default: %(default)s.')
136  parser.add_argument('--debug', action="store_true", default=False, help='Switch on debug mode. Default: %(default)s.')
137  args = parser.parse_args()
138 
139  loglevel = logging.INFO
140  if args.debug:
141  loglevel = logging.DEBUG
142  if args.bookkeeping_db == 'prod':
143  args.bookkeeping_db = 'dev'
144  logging.basicConfig(level=loglevel, format='[%(asctime)s] %(levelname)s: %(message)s')
145 
146  if not args.since.isdigit():
147  raise RuntimeError('Since (=%s) must be a valid run number!'%(args.since))
148 
149  args.destTags = args.destTags.strip().split(',')
150 
151  logging.info('Running O2O %s on machine [%s]' % (args.analyzer, socket.gethostname()))
152 
153  try:
154  jobdirbase = os.environ[jobDirVar]
155  except KeyError:
156  jobdirbase = '/tmp'
157  logging.warning('%s not set in env, will use %s' % (jobDirVar, jobdirbase))
158 
159  # change filepaths in args to abs path
160  args.cfgfile = os.path.abspath(args.cfgfile)
161  if args.skiplistFile:
162  args.skiplistFile = os.path.abspath(args.skiplistFile)
163  if args.whitelistFile:
164  args.whitelistFile = os.path.abspath(args.whitelistFile)
165 
166  # change to O2O working directory
167  jobdir = os.path.join(jobdirbase, args.since, args.analyzer)
168  if not os.path.exists(jobdir):
169  os.makedirs(jobdir)
170  os.chdir(jobdir)
171  logging.info('Running O2O in %s' % jobdir)
172 
173  # run job and upload
174  runjob(args)
175 
176  logging.info('Done!')
177 
178 
179 
double split
Definition: MVATrainer.cc:139
def SiStripDAQPopCon.runjob (   args)

Definition at line 20 of file SiStripDAQPopCon.py.

References f, split, and str.

Referenced by main().

20 def runjob(args):
21  if args.debug:
22  logging.debug(str(args))
23 
24  # read cfglines from input cfgfile
25  with open(args.cfgfile) as cfgfile:
26  cfglines = cfgfile.read()
27  logging.debug(cfglines)
28 
29  # create config from template
30  job_file = 'cfg_{type}_{run}.py'.format(type=args.analyzer, run=args.since)
31  output_db = '{type}_{run}.db'.format(type=args.analyzer, run=args.since)
32  if os.path.exists(output_db):
33  logging.info('Output sqlite file %s already exists! Deleting...' % output_db)
34  os.remove(output_db)
35  hashmap_db = 'hashmap_{type}_{run}.db'.format(type=args.analyzer, run=args.since)
36  if os.path.exists(hashmap_db):
37  logging.info('Hashmap sqlite file %s already exists! Deleting...' % hashmap_db)
38  os.remove(hashmap_db)
39  replace_dict = {'_CFGLINES_' : cfglines.replace('\\', ''),
40  '_ANALYZER_' : args.analyzer,
41  '_USEANALYSIS_':'False',
42  '_CONDDB_' : args.condDbRead,
43  '_DBFILE_' : 'sqlite:///%s' % output_db,
44  '_TARGETTAG_': args.inputTag,
45  '_RUNNUMBER_': args.since,
46  '_HASHMAPDB_': args.hashmapDb,
47  '_MAPDBFILE_': 'sqlite:///%s' % hashmap_db,
48  '_SKIPPED_' : '',
49  '_WHITELISTED_': '',
50  }
51  if args.analyzer == 'SiStripO2OApvGain':
52  # special treatment for G1 O2O
53  skipped = ''
54  if args.skiplistFile:
55  with open(args.skiplistFile) as skipfile:
56  skipped = skipfile.read()
57  else:
58  logging.warning('Skipped module list not provided! No module will be skipped...')
59  whitelisted = ''
60  if args.whitelistFile:
61  with open(args.whitelistFile) as wfile:
62  whitelisted = wfile.read()
63  else:
64  logging.warning('Module whitelist not provided!')
65  replace_dict['_USEANALYSIS_'] = 'True'
66  replace_dict['_SKIPPED_'] = skipped
67  replace_dict['_WHITELISTED_'] = whitelisted
68  replace_dict['_HASHMAPDB_'] = ''
69  replace_dict['_MAPDBFILE_'] = ''
70 
71  # find template cfg file
72  for basedir in os.environ['CMSSW_SEARCH_PATH'].split(':'):
73  templatefile = os.path.join(basedir, cfg_template)
74  if os.path.exists(templatefile):
75  logging.info('Use template config file %s' % templatefile)
76  break
77  config = helper.insert_to_file(templatefile, job_file, replace_dict)
78  logging.info('Start running O2O...')
79  logging.debug(' ... config:\n%s\n' % config)
80 
81  # run cmssw job: raise error if failed
82  command = 'cmsRun %s' % job_file
83  pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
84  atexit.register(partial(helper.kill_subproc_noexcept, pipe))
85  out = pipe.communicate()[0]
86  logging.info('\n%s\n' % out)
87  logging.info('@@@CMSSW job return code = %d@@@' % pipe.returncode)
88  if pipe.returncode != 0:
89  raise RuntimeError('O2O job FAILED!')
90 
91  # upload: raise error if failed
92  if args.no_upload:
93  logging.info('Will not run uploading as requested!')
94  return
95  if args.use_uploader:
96  f = helper.upload_payload
97  else:
98  f = helper.copy_payload
99  f(dbFile=output_db, inputTag=args.inputTag, destTags=args.destTags, destDb=args.destDb, since=args.since,
100  userText='{type}, run: {run}'.format(type=args.analyzer, run=args.since))
101 
102  # post O2O tasks: bookkeeping for fast O2O or G1 O2O
103  if args.analyzer == 'SiStripO2OApvGain':
104  logging.info('Writting bookkeeping info to database.')
105  dbmgr = DbManagerGain(args.bookkeeping_db)
106  dbmgr.update_gain_logs(args.since, job_file)
107  else:
108  logging.info('Updating config-to-payload hash map to database.')
109  dbmgr = DbManagerDAQ(args.bookkeeping_db)
110  dbmgr.update_hashmap(hashmap_db)
111 
112  # clean up
113  try:
114  os.remove(output_db)
115  os.remove(output_db.replace('.db', '.txt'))
116  os.remove(hashmap_db) # may not exist
117  except OSError:
118  pass
119 
double f[11][100]
#define str(s)
double split
Definition: MVATrainer.cc:139