o2olib.O2OJobMgr Class Reference

Inheritance diagram for o2olib.O2OJobMgr:

Public Member Functions
def	__init__ (self, logLevel)
def	add (self, job_name, configJson, int_val, freq_flag, en_flag)
def	connect (self, service, args)
def	dumpConfig (self, jname, versionIndex, configFile)
def	getSession (self, db_service, role, authPath)
def	listConfig (self, jname)
def	listJobs (self)
def	runManager (self)
def	set (self, job_name, en_flag, fr_val=None)
def	setConfig (self, job_name, configJson)
def	setInterval (self, job_name, int_val)

Public Attributes
	conf_dict
	db_connection
	eng
	logger
	session
	verbose

Detailed Description

Definition at line 89 of file o2olib.py.

Constructor & Destructor Documentation

__init__()

def o2olib.O2OJobMgr.__init__	(		self,
			logLevel
	)

Definition at line 91 of file o2olib.py.

    def __init__( self , logLevel):
        self.db_connection = None
        self.conf_dict = {}
        fmt_str = "[%(asctime)s] %(levelname)s: %(message)s"
        if messageLevelEnvVar in os.environ:
            levStr = os.environ[messageLevelEnvVar]
            if levStr == 'DEBUG':
                logLevel = logging.DEBUG
        logFormatter = logging.Formatter(fmt_str)

        self.logger = logging.getLogger()        
        self.logger.setLevel(logLevel)
        consoleHandler = logging.StreamHandler(sys.stdout) 
        consoleHandler.setFormatter(logFormatter)
        self.logger.addHandler(consoleHandler)
        self.eng = None

Member Function Documentation

add()

def o2olib.O2OJobMgr.add	(		self,
			job_name,
			configJson,
			int_val,
			freq_flag,
			en_flag
	)

Definition at line 148 of file o2olib.py.

References relativeConstraints.error, info(), crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, querying.connection.session, and o2olib.O2OJobMgr.session.

Referenced by counter.Counter.register(), SequenceTypes._TaskBase.remove(), SequenceTypes._TaskBase.replace(), and o2olib.O2OJobMgr.setConfig().

    def add( self, job_name, configJson, int_val, freq_flag, en_flag ):
        if configJson == '':
            return False
        res = self.session.query(O2OJob.enabled).filter_by(name=job_name)
        enabled = None
        for r in res:
            enabled = r
        if enabled:
            self.logger.error( "A job called '%s' exists already.", job_name )
            return False
        freq_val = 0
        if freq_flag:
            freq_val = 1
        job = O2OJob(name=job_name,tag_name='-',enabled=en_flag,frequent=freq_val,interval=int_val)
        config = O2OJobConf( job_name=job_name, insertion_time = datetime.utcnow(), configuration = configJson ) 
        self.session.add(job)
        self.session.add(config)
        self.session.commit()
        self.logger.info( "New o2o job '%s' created.", job_name )
        return True 

connect()

def o2olib.O2OJobMgr.connect	(		self,
			service,
			args
	)

Definition at line 136 of file o2olib.py.

Referenced by o2o_db_cfgmap.DbManagerDAQ.update_hashmap().

    def connect( self, service, args ):
        self.session = self.getSession( service, args.role, args.auth )
        self.verbose = args.verbose
        if self.session is None:
            return False
        else:
            self.db_connection = coral_tpl %(service,schema_name)
            self.conf_dict['db']=self.db_connection
            return True

dumpConfig()

def o2olib.O2OJobMgr.dumpConfig	(		self,
			jname,
			versionIndex,
			configFile
	)

Definition at line 268 of file o2olib.py.

References relativeConstraints.error, createfilelist.int, crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, print(), querying.connection.session, o2olib.O2OJobMgr.session, and str.

Referenced by Types.SecSource.configValue().

    def dumpConfig( self, jname, versionIndex, configFile ):
        versionIndex = int(versionIndex)
        res = self.session.query(O2OJob.enabled).filter_by(name=jname)
        enabled = None
        for r in res:
            enabled = r
        if enabled is None:
            self.logger.error( "A job called '%s' does not exist.", jname )
            return
        res = self.session.query( O2OJobConf.configuration, O2OJobConf.insertion_time  ).filter_by(job_name=jname).order_by(O2OJobConf.insertion_time)
        configs = []
        for r in res:
            configs.append((str(r[0]),r[1]))
        ind = len(configs)
        if versionIndex>ind or versionIndex==0:
            self.logger.error("Configuration for job %s with index %s has not been found." %(jname,versionIndex))
            return
        print("Configuration #%2d for job '%s'" %(versionIndex,jname))
        config = configs[versionIndex-1]
        print('#%2d  since %s' %(versionIndex,config[1]))
        print(config[0])
        if configFile is None or configFile == '':
            configFile = '%s_%s.json' %(jname,versionIndex)
        with open(configFile,'w') as json_file:
            json_file.write(config[0])

            

getSession()

def o2olib.O2OJobMgr.getSession	(		self,
			db_service,
			role,
			authPath
	)

Definition at line 108 of file o2olib.py.

References debug, o2olib.O2OJobMgr.eng, relativeConstraints.error, info(), crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, and str.

    def getSession( self, db_service, role, authPath ):
        url = None
        if db_service is None:
            url = private_db
        else:
            self.logger.info('Getting credentials')
            if authPath is not None:
                if not os.path.exists(authPath):
                    self.logger.error('Authentication path %s is invalid.' %authPath)
                    return None
            try:
                (username, account, pwd) = auth.get_credentials_for_schema( db_service, schema_name, role, authPath )
            except Exception as e:
                self.logger.debug(str(e))
                username = None
                pwd = None
            if username is None:
                self.logger.error('Credentials for service %s are not available' %db_service)
                raise Exception("Cannot connect to db %s" %db_service )
            url = sqlalchemy_tpl %(username,pwd,db_service)
        session = None
        try:
            self.eng = sqlalchemy.create_engine( url, max_identifier_length=30)
            session = sqlalchemy.orm.scoped_session( sqlalchemy.orm.sessionmaker(bind=self.eng))
        except sqlalchemy.exc.SQLAlchemyError as dberror:
            self.logger.error( str(dberror) )
        return session

listConfig()

def o2olib.O2OJobMgr.listConfig	(		self,
			jname
	)

Definition at line 246 of file o2olib.py.

References relativeConstraints.error, info(), crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, print(), querying.connection.session, o2olib.O2OJobMgr.session, and str.

    def listConfig( self, jname ):
        res = self.session.query(O2OJob.enabled).filter_by(name=jname)
        enabled = None
        for r in res:
            enabled = r
        if enabled is None:
            self.logger.error( "A job called '%s' does not exist.", jname )
            return
        res = self.session.query( O2OJobConf.configuration, O2OJobConf.insertion_time  ).filter_by(job_name=jname).order_by(O2OJobConf.insertion_time)
        configs = []
        for r in res:
            configs.append((str(r[0]),r[1]))
        ind = len(configs)
        if ind:
            print("Configurations for job '%s'" %jname)
            for cf in reversed(configs):
                print('#%2d  since: %s' %(ind,cf[1]))
                print(cf[0])
                ind -= 1
        else:
            self.logger.info("No configuration found for job '%s'" %jname )

listJobs()

def o2olib.O2OJobMgr.listJobs

(

self

)

Definition at line 223 of file o2olib.py.

References python.cmstools.all(), o2olib.print_table(), querying.connection.session, o2olib.O2OJobMgr.session, and str.

Referenced by o2olib.O2OTool.execute().

    def listJobs( self ):
        runs = {}
        res = self.session.query(O2ORun.job_name,sqlalchemy.func.max(O2ORun.start_time)).group_by(O2ORun.job_name).order_by(O2ORun.job_name)
        for r in res:
            runs[r[0]] = str(r[1])
        res = self.session.query(O2OJob.name, O2OJob.interval, O2OJob.enabled, O2OJob.frequent).order_by(O2OJob.name).all()
        table = []
        for r in res:
            row = []
            row.append(r[0]),
            row.append('%5d' %r[1] )
            frequent = 'Y' if (r[3]==1) else 'N'
            row.append('%4s' %frequent )
            enabled = 'Y' if (r[2]==1) else 'N'
            row.append('%4s' %enabled )
            lastRun = '-'
            if r[0] in runs.keys():
                lastRun = runs[r[0]]
            row.append( lastRun )
            table.append(row)
        headers = ['Job name','Interval','Frequent','Enabled','Last run start']
        print_table( headers, table ) 

runManager()

def o2olib.O2OJobMgr.runManager

(

self

)

Definition at line 145 of file o2olib.py.

References o2olib.O2OJobMgr.db_connection, crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, querying.connection.session, and o2olib.O2OJobMgr.session.

    def runManager( self ):
        return O2ORunMgr( self.db_connection, self.session, self.logger )

set()

def o2olib.O2OJobMgr.set	(		self,
			job_name,
			en_flag,
			fr_val = None
	)

Definition at line 169 of file o2olib.py.

References relativeConstraints.error, info(), crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, querying.connection.session, and o2olib.O2OJobMgr.session.

    def set( self, job_name, en_flag, fr_val=None ):
        res = self.session.query(O2OJob.enabled).filter_by(name=job_name)
        enabled = None
        for r in res:
            enabled = r
        if enabled is None:
            self.logger.error( "A job called '%s' does not exist.", job_name )
            return
        if en_flag is not None and enabled != en_flag:
            job = O2OJob(name=job_name,enabled=en_flag)
            self.session.merge(job)
            self.session.commit()
            action = 'enabled'
            if not en_flag:
                action = 'disabled'
            self.logger.info( "Job '%s' %s." %(job_name,action) )
        if fr_val is not None:
            job = O2OJob(name=job_name,frequent=fr_val)
            self.session.merge(job)
            self.session.commit()
            if fr_val==1:
                self.logger.info( "Job '%s' set 'frequent'" %job_name)
            else:
                self.logger.info( "Job '%s' unset 'frequent'" %job_name)

setConfig()

def o2olib.O2OJobMgr.setConfig	(		self,
			job_name,
			configJson
	)

Definition at line 194 of file o2olib.py.

References o2olib.O2OJobMgr.add(), relativeConstraints.error, info(), crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, querying.connection.session, and o2olib.O2OJobMgr.session.

Referenced by o2olib.O2OTool.execute().

    def setConfig( self, job_name, configJson ):
        if configJson == '':
            return False
        res = self.session.query(O2OJob.enabled).filter_by(name=job_name)
        enabled = None
        for r in res:
            enabled = r
        if enabled is None:
            self.logger.error( "A job called '%s' does not exist.", job_name )
            return
        config = O2OJobConf( job_name=job_name, insertion_time = datetime.utcnow(), configuration = configJson )     
        self.session.add(config)
        self.session.commit()
        self.logger.info( "New configuration inserted for job '%s'", job_name )
        return True 

setInterval()

def o2olib.O2OJobMgr.setInterval	(		self,
			job_name,
			int_val
	)

Definition at line 210 of file o2olib.py.

References relativeConstraints.error, info(), crabFunctions.CrabController.logger, o2olib.O2OJobMgr.logger, querying.connection.session, and o2olib.O2OJobMgr.session.

Referenced by o2olib.O2OTool.execute().

    def setInterval( self, job_name, int_val ):
        res = self.session.query(O2OJob.enabled).filter_by(name=job_name)
        enabled = None
        for r in res:
            enabled = r
        if enabled is None:
            self.logger.error( "A job called '%s' does not exist.", job_name )
            return
        job = O2OJob(name=job_name,interval=int_val)
        self.session.merge(job)
        self.session.commit()
        self.logger.info( "The execution interval for job '%s' has been updated.", job_name )

Member Data Documentation

conf_dict

o2olib.O2OJobMgr.conf_dict

Definition at line 93 of file o2olib.py.

Referenced by o2olib.O2ORunMgr.executeJob().

db_connection

o2olib.O2OJobMgr.db_connection

Definition at line 92 of file o2olib.py.

Referenced by o2olib.O2OJobMgr.runManager().

eng

o2olib.O2OJobMgr.eng

Definition at line 106 of file o2olib.py.

Referenced by o2olib.O2OJobMgr.getSession().

logger

o2olib.O2OJobMgr.logger

Definition at line 101 of file o2olib.py.

Referenced by o2olib.O2OJobMgr.add(), o2olib.O2OJobMgr.dumpConfig(), o2olib.O2ORunMgr.endJob(), o2olib.O2ORunMgr.executeJob(), o2olib.O2OJobMgr.getSession(), o2olib.O2OJobMgr.listConfig(), looper.Looper.loop(), cmsHarvester.CMSHarvester.option_handler_book_keeping_file(), cmsHarvester.CMSHarvester.option_handler_debug(), cmsHarvester.CMSHarvester.option_handler_force(), cmsHarvester.CMSHarvester.option_handler_frontier_connection(), cmsHarvester.CMSHarvester.option_handler_globaltag(), cmsHarvester.CMSHarvester.option_handler_harvesting_mode(), cmsHarvester.CMSHarvester.option_handler_harvesting_type(), cmsHarvester.CMSHarvester.option_handler_input_spec(), cmsHarvester.CMSHarvester.option_handler_no_ref_hists(), cmsHarvester.CMSHarvester.option_handler_ref_hist_mapping_file(), o2olib.O2OJobMgr.runManager(), o2olib.O2OJobMgr.set(), o2olib.O2OJobMgr.setConfig(), o2olib.O2OJobMgr.setInterval(), and o2olib.O2ORunMgr.startJob().

session

o2olib.O2OJobMgr.session

Definition at line 137 of file o2olib.py.

Referenced by o2olib.O2OJobMgr.add(), o2olib.O2OJobMgr.dumpConfig(), o2olib.O2ORunMgr.endJob(), o2olib.O2OJobMgr.listConfig(), o2olib.O2OJobMgr.listJobs(), o2olib.O2OJobMgr.runManager(), o2olib.O2OJobMgr.set(), o2olib.O2OJobMgr.setConfig(), o2olib.O2OJobMgr.setInterval(), and o2olib.O2ORunMgr.startJob().

verbose

o2olib.O2OJobMgr.verbose

Definition at line 138 of file o2olib.py.

Referenced by python.diff_provenance.difference.list_diff(), python.diffProv.difference.list_diff(), python.diff_provenance.difference.module_diff(), python.diffProv.difference.module_diff(), python.diff_provenance.difference.onefilemodules(), python.diffProv.difference.onefilemodules(), core.TriggerMatchAnalyzer.TriggerMatchAnalyzer.process(), core.SkimAnalyzerCount.SkimAnalyzerCount.process(), objects.VertexAnalyzer.VertexAnalyzer.process(), and confdbOfflineConverter.OfflineConverter.query().