CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
List of all members | Public Member Functions | Public Attributes
o2olib.O2OJobMgr Class Reference
Inheritance diagram for o2olib.O2OJobMgr:

Public Member Functions

def __init__ (self, logLevel)
 
def add (self, job_name, configJson, int_val, freq_flag, en_flag)
 
def connect (self, service, args)
 
def dumpConfig (self, jname, versionIndex, configFile)
 
def getSession (self, db_service, role, authPath)
 
def listConfig (self, jname)
 
def listJobs (self)
 
def runManager (self)
 
def set (self, job_name, en_flag, fr_val=None)
 
def setConfig (self, job_name, configJson)
 
def setInterval (self, job_name, int_val)
 

Public Attributes

 conf_dict
 
 db_connection
 
 eng
 
 logger
 
 session
 
 verbose
 

Detailed Description

Definition at line 89 of file o2olib.py.

Constructor & Destructor Documentation

◆ __init__()

def o2olib.O2OJobMgr.__init__ (   self,
  logLevel 
)

Definition at line 91 of file o2olib.py.

91  def __init__( self , logLevel):
92  self.db_connection = None
93  self.conf_dict = {}
94  fmt_str = "[%(asctime)s] %(levelname)s: %(message)s"
95  if messageLevelEnvVar in os.environ:
96  levStr = os.environ[messageLevelEnvVar]
97  if levStr == 'DEBUG':
98  logLevel = logging.DEBUG
99  logFormatter = logging.Formatter(fmt_str)
100 
101  self.logger = logging.getLogger()
102  self.logger.setLevel(logLevel)
103  consoleHandler = logging.StreamHandler(sys.stdout)
104  consoleHandler.setFormatter(logFormatter)
105  self.logger.addHandler(consoleHandler)
106  self.eng = None
107 
def __init__(self, dataset, job_number, job_id, job_name, isDA, isMC, applyBOWS, applyEXTRACOND, extraconditions, runboundary, lumilist, intlumi, maxevents, gt, allFromGT, alignmentDB, alignmentTAG, apeDB, apeTAG, bowDB, bowTAG, vertextype, tracktype, refittertype, ttrhtype, applyruncontrol, ptcut, CMSSW_dir, the_dir)

Member Function Documentation

◆ add()

def o2olib.O2OJobMgr.add (   self,
  job_name,
  configJson,
  int_val,
  freq_flag,
  en_flag 
)

Definition at line 148 of file o2olib.py.

References relativeConstraints.error, info(), crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, querying.connection.session, and o2olib.O2OJobMgr.session.

Referenced by counter.Counter.register(), SequenceTypes._TaskBase.remove(), SequenceTypes._TaskBase.replace(), and o2olib.O2OJobMgr.setConfig().

148  def add( self, job_name, configJson, int_val, freq_flag, en_flag ):
149  if configJson == '':
150  return False
151  res = self.session.query(O2OJob.enabled).filter_by(name=job_name)
152  enabled = None
153  for r in res:
154  enabled = r
155  if enabled:
156  self.logger.error( "A job called '%s' exists already.", job_name )
157  return False
158  freq_val = 0
159  if freq_flag:
160  freq_val = 1
161  job = O2OJob(name=job_name,tag_name='-',enabled=en_flag,frequent=freq_val,interval=int_val)
162  config = O2OJobConf( job_name=job_name, insertion_time = datetime.utcnow(), configuration = configJson )
163  self.session.add(job)
164  self.session.add(config)
165  self.session.commit()
166  self.logger.info( "New o2o job '%s' created.", job_name )
167  return True
168 
static const TGPicture * info(bool iBackgroundIsBlack)
Definition: query.py:1
void add(std::map< std::string, TH1 *> &h, TH1 *hist)

◆ connect()

def o2olib.O2OJobMgr.connect (   self,
  service,
  args 
)

Definition at line 136 of file o2olib.py.

Referenced by o2o_db_cfgmap.DbManagerDAQ.update_hashmap().

136  def connect( self, service, args ):
137  self.session = self.getSession( service, args.role, args.auth )
138  self.verbose = args.verbose
139  if self.session is None:
140  return False
141  else:
142  self.db_connection = coral_tpl %(service,schema_name)
143  self.conf_dict['db']=self.db_connection
144  return True

◆ dumpConfig()

def o2olib.O2OJobMgr.dumpConfig (   self,
  jname,
  versionIndex,
  configFile 
)

Definition at line 268 of file o2olib.py.

References relativeConstraints.error, createfilelist.int, crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, print(), querying.connection.session, o2olib.O2OJobMgr.session, and str.

Referenced by Types.SecSource.configValue().

268  def dumpConfig( self, jname, versionIndex, configFile ):
269  versionIndex = int(versionIndex)
270  res = self.session.query(O2OJob.enabled).filter_by(name=jname)
271  enabled = None
272  for r in res:
273  enabled = r
274  if enabled is None:
275  self.logger.error( "A job called '%s' does not exist.", jname )
276  return
277  res = self.session.query( O2OJobConf.configuration, O2OJobConf.insertion_time ).filter_by(job_name=jname).order_by(O2OJobConf.insertion_time)
278  configs = []
279  for r in res:
280  configs.append((str(r[0]),r[1]))
281  ind = len(configs)
282  if versionIndex>ind or versionIndex==0:
283  self.logger.error("Configuration for job %s with index %s has not been found." %(jname,versionIndex))
284  return
285  print("Configuration #%2d for job '%s'" %(versionIndex,jname))
286  config = configs[versionIndex-1]
287  print('#%2d since %s' %(versionIndex,config[1]))
288  print(config[0])
289  if configFile is None or configFile == '':
290  configFile = '%s_%s.json' %(jname,versionIndex)
291  with open(configFile,'w') as json_file:
292  json_file.write(config[0])
293 
294 
Definition: query.py:1
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
#define str(s)

◆ getSession()

def o2olib.O2OJobMgr.getSession (   self,
  db_service,
  role,
  authPath 
)

Definition at line 108 of file o2olib.py.

References debug, o2olib.O2OJobMgr.eng, relativeConstraints.error, info(), crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, and str.

108  def getSession( self, db_service, role, authPath ):
109  url = None
110  if db_service is None:
111  url = private_db
112  else:
113  self.logger.info('Getting credentials')
114  if authPath is not None:
115  if not os.path.exists(authPath):
116  self.logger.error('Authentication path %s is invalid.' %authPath)
117  return None
118  try:
119  (username, account, pwd) = auth.get_credentials_for_schema( db_service, schema_name, role, authPath )
120  except Exception as e:
121  self.logger.debug(str(e))
122  username = None
123  pwd = None
124  if username is None:
125  self.logger.error('Credentials for service %s are not available' %db_service)
126  raise Exception("Cannot connect to db %s" %db_service )
127  url = sqlalchemy_tpl %(username,pwd,db_service)
128  session = None
129  try:
130  self.eng = sqlalchemy.create_engine( url, max_identifier_length=30)
131  session = sqlalchemy.orm.scoped_session( sqlalchemy.orm.sessionmaker(bind=self.eng))
132  except sqlalchemy.exc.SQLAlchemyError as dberror:
133  self.logger.error( str(dberror) )
134  return session
135 
static const TGPicture * info(bool iBackgroundIsBlack)
#define debug
Definition: HDRShower.cc:19
#define str(s)

◆ listConfig()

def o2olib.O2OJobMgr.listConfig (   self,
  jname 
)

Definition at line 246 of file o2olib.py.

References relativeConstraints.error, info(), crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, print(), querying.connection.session, o2olib.O2OJobMgr.session, and str.

246  def listConfig( self, jname ):
247  res = self.session.query(O2OJob.enabled).filter_by(name=jname)
248  enabled = None
249  for r in res:
250  enabled = r
251  if enabled is None:
252  self.logger.error( "A job called '%s' does not exist.", jname )
253  return
254  res = self.session.query( O2OJobConf.configuration, O2OJobConf.insertion_time ).filter_by(job_name=jname).order_by(O2OJobConf.insertion_time)
255  configs = []
256  for r in res:
257  configs.append((str(r[0]),r[1]))
258  ind = len(configs)
259  if ind:
260  print("Configurations for job '%s'" %jname)
261  for cf in reversed(configs):
262  print('#%2d since: %s' %(ind,cf[1]))
263  print(cf[0])
264  ind -= 1
265  else:
266  self.logger.info("No configuration found for job '%s'" %jname )
267 
static const TGPicture * info(bool iBackgroundIsBlack)
Definition: query.py:1
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
#define str(s)

◆ listJobs()

def o2olib.O2OJobMgr.listJobs (   self)

Definition at line 223 of file o2olib.py.

References python.cmstools.all(), o2olib.print_table(), querying.connection.session, o2olib.O2OJobMgr.session, and str.

Referenced by o2olib.O2OTool.execute().

223  def listJobs( self ):
224  runs = {}
225  res = self.session.query(O2ORun.job_name,sqlalchemy.func.max(O2ORun.start_time)).group_by(O2ORun.job_name).order_by(O2ORun.job_name)
226  for r in res:
227  runs[r[0]] = str(r[1])
228  res = self.session.query(O2OJob.name, O2OJob.interval, O2OJob.enabled, O2OJob.frequent).order_by(O2OJob.name).all()
229  table = []
230  for r in res:
231  row = []
232  row.append(r[0]),
233  row.append('%5d' %r[1] )
234  frequent = 'Y' if (r[3]==1) else 'N'
235  row.append('%4s' %frequent )
236  enabled = 'Y' if (r[2]==1) else 'N'
237  row.append('%4s' %enabled )
238  lastRun = '-'
239  if r[0] in runs.keys():
240  lastRun = runs[r[0]]
241  row.append( lastRun )
242  table.append(row)
243  headers = ['Job name','Interval','Frequent','Enabled','Last run start']
244  print_table( headers, table )
245 
def all(container)
workaround iterator generators for ROOT classes
Definition: cmstools.py:25
Definition: query.py:1
def print_table(headers, table)
Definition: o2olib.py:58
#define str(s)

◆ runManager()

def o2olib.O2OJobMgr.runManager (   self)

Definition at line 145 of file o2olib.py.

References o2olib.O2OJobMgr.db_connection, crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, querying.connection.session, and o2olib.O2OJobMgr.session.

145  def runManager( self ):
146  return O2ORunMgr( self.db_connection, self.session, self.logger )
147 

◆ set()

def o2olib.O2OJobMgr.set (   self,
  job_name,
  en_flag,
  fr_val = None 
)

Definition at line 169 of file o2olib.py.

References relativeConstraints.error, info(), crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, querying.connection.session, and o2olib.O2OJobMgr.session.

169  def set( self, job_name, en_flag, fr_val=None ):
170  res = self.session.query(O2OJob.enabled).filter_by(name=job_name)
171  enabled = None
172  for r in res:
173  enabled = r
174  if enabled is None:
175  self.logger.error( "A job called '%s' does not exist.", job_name )
176  return
177  if en_flag is not None and enabled != en_flag:
178  job = O2OJob(name=job_name,enabled=en_flag)
179  self.session.merge(job)
180  self.session.commit()
181  action = 'enabled'
182  if not en_flag:
183  action = 'disabled'
184  self.logger.info( "Job '%s' %s." %(job_name,action) )
185  if fr_val is not None:
186  job = O2OJob(name=job_name,frequent=fr_val)
187  self.session.merge(job)
188  self.session.commit()
189  if fr_val==1:
190  self.logger.info( "Job '%s' set 'frequent'" %job_name)
191  else:
192  self.logger.info( "Job '%s' unset 'frequent'" %job_name)
193 
Definition: merge.py:1
static const TGPicture * info(bool iBackgroundIsBlack)
Definition: query.py:1

◆ setConfig()

def o2olib.O2OJobMgr.setConfig (   self,
  job_name,
  configJson 
)

Definition at line 194 of file o2olib.py.

References o2olib.O2OJobMgr.add(), relativeConstraints.error, info(), crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, querying.connection.session, and o2olib.O2OJobMgr.session.

Referenced by o2olib.O2OTool.execute().

194  def setConfig( self, job_name, configJson ):
195  if configJson == '':
196  return False
197  res = self.session.query(O2OJob.enabled).filter_by(name=job_name)
198  enabled = None
199  for r in res:
200  enabled = r
201  if enabled is None:
202  self.logger.error( "A job called '%s' does not exist.", job_name )
203  return
204  config = O2OJobConf( job_name=job_name, insertion_time = datetime.utcnow(), configuration = configJson )
205  self.session.add(config)
206  self.session.commit()
207  self.logger.info( "New configuration inserted for job '%s'", job_name )
208  return True
209 
static const TGPicture * info(bool iBackgroundIsBlack)
Definition: query.py:1
void add(std::map< std::string, TH1 *> &h, TH1 *hist)

◆ setInterval()

def o2olib.O2OJobMgr.setInterval (   self,
  job_name,
  int_val 
)

Definition at line 210 of file o2olib.py.

References relativeConstraints.error, info(), crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, querying.connection.session, and o2olib.O2OJobMgr.session.

Referenced by o2olib.O2OTool.execute().

210  def setInterval( self, job_name, int_val ):
211  res = self.session.query(O2OJob.enabled).filter_by(name=job_name)
212  enabled = None
213  for r in res:
214  enabled = r
215  if enabled is None:
216  self.logger.error( "A job called '%s' does not exist.", job_name )
217  return
218  job = O2OJob(name=job_name,interval=int_val)
219  self.session.merge(job)
220  self.session.commit()
221  self.logger.info( "The execution interval for job '%s' has been updated.", job_name )
222 
Definition: merge.py:1
static const TGPicture * info(bool iBackgroundIsBlack)
Definition: query.py:1

Member Data Documentation

◆ conf_dict

o2olib.O2OJobMgr.conf_dict

Definition at line 93 of file o2olib.py.

Referenced by o2olib.O2ORunMgr.executeJob().

◆ db_connection

o2olib.O2OJobMgr.db_connection

Definition at line 92 of file o2olib.py.

Referenced by o2olib.O2OJobMgr.runManager().

◆ eng

o2olib.O2OJobMgr.eng

Definition at line 106 of file o2olib.py.

Referenced by o2olib.O2OJobMgr.getSession().

◆ logger

o2olib.O2OJobMgr.logger

◆ session

o2olib.O2OJobMgr.session

◆ verbose

o2olib.O2OJobMgr.verbose