crabFunctions.CrabController Class Reference

The CrabController class. More...

Public Member Functions
def	__init__ (self, debug=0, logger=None, workingArea=None, voGroup=None, username=None)
	The constructor. More...
def	callCrabCommand (self, crabArgs)
	Call crab command in a new process and return result dict. More...
def	checkusername (self)
	Returns the hn name for a user with valid proxy. More...
def	checkwrite (self, site='T2_DE_RWTH', path='noPath')
	Check if crab can write to specified site. More...
def	commandlineOptions (self, parser=optparse.OptionParser( 'usage:%prog'))
	Populates an existing optparse parser or returns a new one with options for crab functions. More...
def	crabFolders (self)
	Return list of all crab folders in workin area (default cwd) More...
def	getlog (self, name)
	Call crab getlog. More...
def	readCrabConfig (self, name)
	Read a crab config and return python object. More...
def	report (self, name)
	Call crab report command and return path to lumiSummary. More...
def	resubmit (self, name, joblist=None)
	Resubmit all failed tasks in job or specified list of jobs in task. More...
def	status (self, name)
	Check crab status. More...
def	submit (self, name)
	Check if crab can write to specified site. More...

Public Attributes
	crab_q
	debug
	dry_run
	logger
	username
	voGroup
	workingArea

Private Member Functions
def	_prepareFoldername (self, name)
	Add crab_ to Foldername if needed. More...

Detailed Description

The CrabController class.

This class can be used to manage Analyses using crab3

Definition at line 31 of file crabFunctions.py.

Constructor & Destructor Documentation

__init__()

def crabFunctions.CrabController.__init__	(		self,
			debug = 0,
			logger = None,
			workingArea = None,
			voGroup = None,
			username = None
	)

The constructor.

Parameters

self	The object pointer.
self	A previously defined logger. Crab log messages will use this logger as their parent logger.

Definition at line 36 of file crabFunctions.py.

    def __init__(self, debug=0, logger = None , workingArea = None, voGroup = None, username = None):
 
        setConsoleLogLevel(LOGLEVEL_MUTE)
        self.debug = debug
        if workingArea is not None:
            self.workingArea = workingArea
        else:
            self.workingArea = os.getcwd()
        self.dry_run = False
        if voGroup is not None:
            self.voGroup = voGroup
        else:
            self.voGroup = "dcms"
        if username is not None:
            self.username = username
        else:
            self.username = None
 
        if logger is not None:
            self.logger = logger.getChild("CrabController")
        else:
            # add instance logger as logger to root
            self.logger = logging.getLogger("CrabController")
            # check if handlers are present for root logger
            # we assume that the default logging is not configured
            # if handler is present
            if len(logging.getLogger().handlers) < 1 :
                ch = logging.FileHandler('crabController.log', mode='a', encoding=None, delay=False)
                ch.setLevel(logging.DEBUG)
                # create formatter
                formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s' )
                # add formatter to ch
                ch.setFormatter(formatter)
                self.logger.addHandler(ch)
 
        self.crab_q = Queue()

Member Function Documentation

_prepareFoldername()

def crabFunctions.CrabController._prepareFoldername (   self,   name  )

private

Add crab_ to Foldername if needed.

Parameters

getlog(self,name)

Definition at line 255 of file crabFunctions.py.

    def _prepareFoldername(self, name):
        if name.startswith("crab_"):
            crabfolder = '%s'%name
        else:
            crabfolder = "crab_%s "%name
        return crabfolder.strip()

Referenced by crabFunctions.CrabController.getlog(), crabFunctions.CrabController.report(), and crabFunctions.CrabController.resubmit().

callCrabCommand()

def crabFunctions.CrabController.callCrabCommand	(		self,
			crabArgs
	)

Call crab command in a new process and return result dict.

Parameters

self	The object pointer
crabArgs	A list of arguments for crab beginning with the command

Definition at line 189 of file crabFunctions.py.

    def callCrabCommand( self, crabArgs ):
        crabCommandProcessArgs = (self.crab_q, crabArgs)
        p = Process(target=crabCommandProcess, args=(crabCommandProcessArgs))
        p.start()
        res = self.crab_q.get()
        p.join()
        return res
 

References crabFunctions.CrabController.crab_q.

Referenced by crabFunctions.CrabController.getlog(), crabFunctions.CrabController.report(), crabFunctions.CrabController.resubmit(), crabFunctions.CrabController.status(), and crabFunctions.CrabController.submit().

checkusername()

def crabFunctions.CrabController.checkusername

(

self

)

Returns the hn name for a user with valid proxy.

@type self: CrabController

Parameters

self	The object pointer.

Returns

users hypernews name

Definition at line 141 of file crabFunctions.py.

    def checkusername(self):
        #depreceated string: cmd = 'crab checkHNname --voGroup=dcms'
        #~ cmd = 'crab checkusername --voGroup=dcms'
        try:
            username = os.environ["CERNUSERNAME"]
            return username
        except:pass
        res = crabCommand('checkusername')
        try:
            self.username = res['username']
            return res['username']
        except:
            return "noHNname"
 

References crabFunctions.CrabController.username.

Referenced by crabFunctions.CrabController.checkwrite().

checkwrite()

def crabFunctions.CrabController.checkwrite	(		self,
			site = 'T2_DE_RWTH',
			path = 'noPath'
	)

Check if crab can write to specified site.

@type self: CrabController

Parameters

self	The object pointer. @type site string
site	The Site symbol [default:T2_DE_RWTH] @type path string
path	lfn path to check write permission in. see twiki WorkBookCRAB3Tutorial

Returns

boolean which is True if user can write to site and False otherwise

Definition at line 81 of file crabFunctions.py.

    def checkwrite(self,site='T2_DE_RWTH',path='noPath'):
        if self.username is None: self.checkusername()
        try:
            self.logger.info( "Checking if user can write to /store/user/%s on site %s with voGroup %s"%(self.username,site , self.voGroup) )
            if not 'noPath' in path:
                res = crabCommand('checkwrite','--site',site,'--voGroup',self.voGroup,'--lfn', path)
            else:
                res = crabCommand('checkwrite','--site',site,'--voGroup',self.voGroup)
            if res['status'] == 'SUCCESS':
                self.logger.info("Checkwrite was sucessfully called.")
                return True
            else:
                self.logger.error( "The crab checkwrite command failed for site: %s"%site )
                return False
        except:
            self.logger.error( 'Unable to perform crab checkwrite')
            return False
 

References crabFunctions.CrabController.checkusername(), relativeConstraints.error, info(), crabFunctions.CrabController.logger, crabFunctions.CrabController.username, and crabFunctions.CrabController.voGroup.

commandlineOptions()

def crabFunctions.CrabController.commandlineOptions	(		self,
			parser = optparse.OptionParser( 'usage: %prog' )
	)

Populates an existing optparse parser or returns a new one with options for crab functions.

This functions populates a previously created (or new) instance of a optparse parser object with options needed by crab functions. It is possible to add three kinds of options:

• options where a error should be raised if the option was previously defined
• options where previous definitions should be kept
• options where previous definitions should be overriden @type Optparse parser instance

  Parameters

  parser

  A previously created parser oject which should be extenden [default: new instance]

  Returns

  A new or extenden optparse parser instance

Definition at line 272 of file crabFunctions.py.

    def commandlineOptions(self,parser = optparse.OptionParser( 'usage: %prog' )):
        # we first need to call parse_args with a dummy string at the beginning to
        # check for existing options later
        (currentoptions, args ) = parser.parse_args([" "])
 
        # The following block shows how variables should be added, where
        # conflicts are possible if the option is overridden by this function
        # they raise a value error
        #try:
        #    parser.add_option( '--someOption',metavar='DIR',default=None,
        #                       help='Dummy option for future integration')
        #except OptionConflictError as e:
        #    conditionalLog(crablog,"There are conflicts extending the optparse options object",'error')
        #    conditionalLog(crablog,e.strerror,'error')
 
        # options where it is checked if they exists and new options are added
        # otherwise
        if not hasattr(currentoptions, 'dry_run'):
            parser.add_option( '--dry-run', action='store_true', default=False,
                            help='Do everything except calling CRAB or registering samples to the database.' )
        if not hasattr(currentoptions, 'workingArea'):
            parser.add_option( '--workingArea',metavar='DIR',default=os.getcwd(),help='The area (full or relative path) where to create the CRAB project directory. '
                                 'If the area doesn\'t exist, CRAB will try to create it using the mkdir command' \
                                ' (without -p option). Defaults to the current working directory.'       )
 
 
        # Some options can be added without expected trouble with other parser
        # parts, simply because it is quite fixed what they represent.
        # those variables should be added here and will throw no exception if
        # they already exist in the parser
        #parser.set_conflict_handler('resolve')
        #parser.add_option( '--someOption',metavar='DIR',default=None,
        #                       help='Dummy option for future integration')
 
        return parser
 
 
 

crabFolders()

def crabFunctions.CrabController.crabFolders

(

self

)

Return list of all crab folders in workin area (default cwd)

Parameters

self	The object pointer

Definition at line 247 of file crabFunctions.py.

    def crabFolders(self):
        results = []
        dirlist = [ x for x in os.listdir( self.workingArea ) if (x.startswith('crab_') and os.path.isdir( os.path.join(self.workingArea,x) ) )]
        return dirlist
 

References crabFunctions.CrabController.workingArea.

getlog()

def crabFunctions.CrabController.getlog	(		self,
			name
	)

Call crab getlog.

Parameters

self	The object pointer. @type name string

Definition at line 201 of file crabFunctions.py.

    def getlog(self, name):
        foldername = self._prepareFoldername( name)
        try:
            #res = crabCommand('--quiet','status', dir = 'crab_%s' % name)
            res = self.callCrabCommand( ('getlog',  '%s' % foldername) )
            return res['success'], res['failed']
        except:
            self.logger.error("Error calling crab getlog for %s" %foldername)
            return {}, {}
 

References crabFunctions.CrabController._prepareFoldername(), crabFunctions.CrabController.callCrabCommand(), relativeConstraints.error, and crabFunctions.CrabController.logger.

readCrabConfig()

def crabFunctions.CrabController.readCrabConfig	(		self,
			name
	)

Read a crab config and return python object.

Parameters

self	The object pointer.
name	The sample name (crab request name)

Definition at line 228 of file crabFunctions.py.

    def readCrabConfig( self, name ):
        try:
            if os.path.exists(name):
                pset = name
            else:
                pset = 'crab_%s_cfg.py' % name
            with open( pset, 'r') as cfgfile:
                cfo = imp.load_source("pycfg", pset, cfgfile )
                config = cfo.config
                del cfo
            return config
        except:
            return False
 

report()

def crabFunctions.CrabController.report	(		self,
			name
	)

Call crab report command and return path to lumiSummary.

Parameters

self	The object pointer
name	The crab3 request name, a.k.a the sample name

Definition at line 215 of file crabFunctions.py.

    def report(self, name):
        foldername = self._prepareFoldername( name)
        try:
            res = self.callCrabCommand( ('report',  '%s' % foldername) )
            return res['analyzedLumis']
        except:
            self.logger.error("Error calling crab report for %s" %foldername)
 
 

References crabFunctions.CrabController._prepareFoldername(), crabFunctions.CrabController.callCrabCommand(), relativeConstraints.error, and crabFunctions.CrabController.logger.

resubmit()

def crabFunctions.CrabController.resubmit	(		self,
			name,
			joblist = None
	)

Resubmit all failed tasks in job or specified list of jobs in task.

@type self: CrabController

Parameters

self	The object pointer. @type name string
name	The crab3 request name, a.k.a the sample name @type joblist list of strings
joblist	The crab3 request name, a.k.a the sample name

Definition at line 122 of file crabFunctions.py.

    def resubmit(self,name,joblist = None):
        if self.dry_run:
            self.logger.info('Dry-run: Created config file. ')
            return {}
        #~ if joblist is not None:
            #~ jobstring ="%s"%','.join(joblist)
            #~ cmd = ('resubmit','--wait', '--jobids=',jobstring, os.path.join(self.workingArea,self._prepareFoldername(name)) )
        if False:
            pass
        else:
            cmd = ('resubmit','--wait', os.path.join(self.workingArea,self._prepareFoldername(name)) )
        res = self.callCrabCommand( cmd )
        self.logger.info("crab resumbit called for task %s"%name)
        return res

References crabFunctions.CrabController._prepareFoldername(), crabFunctions.CrabController.callCrabCommand(), crabFunctions.CrabController.dry_run, info(), crabFunctions.CrabController.logger, and crabFunctions.CrabController.workingArea.

status()

def crabFunctions.CrabController.status	(		self,
			name
	)

Check crab status.

@type self: CrabController

Parameters

self	The object pointer. @type name string
name	The crab3 request name, a.k.a the sample name

Definition at line 161 of file crabFunctions.py.

    def status(self,name):
        if self.dry_run:
            self.logger.info('Dry-run: Created config file. crab command would have been: %s'%cmd)
        else:
            try:
                if not "crab_" in name:
                    callname = "crab_" + name
                else:
                    callname = name
                res = self.callCrabCommand( ('status', '--long', callname) )
                #print res
                if 'taskFailureMsg' in res and 'jobs' in res:
                    return res['status'], res['jobs'], res['taskFailureMsg']
                elif 'jobs' in res and 'taskFailureMsg' not in res:
                    return res['status'], res['jobs'],None
                elif 'jobs' not in res and 'taskFailureMsg' in res:
                    return res['status'], {},res['taskFailureMsg']
                else:
                     return res['status'],{},None
            except Exception as e:
                print(e)
                self.logger.error("Can not run crab status request")
                return "NOSTATE",{},None
 

References crabFunctions.CrabController.callCrabCommand(), crabFunctions.CrabController.dry_run, relativeConstraints.error, info(), crabFunctions.CrabController.logger, and edm.print().

submit()

def crabFunctions.CrabController.submit	(		self,
			name
	)

Check if crab can write to specified site.

Parameters

self	The object pointer. @type name string
name	The crab3 config file name

Definition at line 104 of file crabFunctions.py.

    def submit(self,name):
        if self.dry_run:
            res = self.callCrabCommand(('submit', '--dryrun', name))
            self.logger.info('Dry-run: You may check the created config and sandbox')
        else:
            res = self.callCrabCommand(('submit','--wait' , name))
            self.logger.info("crab sumbit called for task %s"%name)
            if self.debug > 1:
                self.logger.info(str(res))
        return res

References crabFunctions.CrabController.callCrabCommand(), ALIUtils.debug, crabFunctions.CrabController.debug, DTTTrigCorrectionFirst.debug, DTTPDeadWriter.debug, DTTTrigWriter.debug, DTT0CalibrationRMS.debug, DTNoiseComputation.debug, DTT0Calibration.debug, DTTTrigCalibration.debug, DTTMax.debug, DTVDriftCalibration.debug, crabFunctions.CrabController.dry_run, info(), crabFunctions.CrabController.logger, and str.

Member Data Documentation

crab_q

crabFunctions.CrabController.crab_q

Definition at line 71 of file crabFunctions.py.

Referenced by crabFunctions.CrabController.callCrabCommand().

debug

crabFunctions.CrabController.debug

Definition at line 39 of file crabFunctions.py.

Referenced by util.rrapi.RRApi.dprint(), rrapi.RRApi.dprint(), pkg.AbstractPkg.generate(), rrapi.RRApi.get(), util.rrapi.RRApi.get(), pkg.AbstractPkg.get_kwds(), runTauIdMVA.TauIDEmbedder.loadMVA_WPs_run2_2017(), runTauIdMVA.TauIDEmbedder.runTauID(), crabFunctions.CrabController.submit(), and pkg.AbstractPkg.write().

dry_run

crabFunctions.CrabController.dry_run

Definition at line 44 of file crabFunctions.py.

Referenced by crabFunctions.CrabController.resubmit(), crabFunctions.CrabController.status(), and crabFunctions.CrabController.submit().

logger

crabFunctions.CrabController.logger

Definition at line 55 of file crabFunctions.py.

Referenced by crabFunctions.CrabController.checkwrite(), o2olib.O2ORunMgr.endJob(), o2olib.O2ORunMgr.executeJob(), crabFunctions.CrabController.getlog(), o2olib.O2OMgr.getSession(), o2olib.O2OMgr.logger(), looper.Looper.loop(), cmsHarvester.CMSHarvester.option_handler_book_keeping_file(), cmsHarvester.CMSHarvester.option_handler_debug(), cmsHarvester.CMSHarvester.option_handler_force(), cmsHarvester.CMSHarvester.option_handler_frontier_connection(), cmsHarvester.CMSHarvester.option_handler_globaltag(), cmsHarvester.CMSHarvester.option_handler_harvesting_mode(), cmsHarvester.CMSHarvester.option_handler_harvesting_type(), cmsHarvester.CMSHarvester.option_handler_input_spec(), cmsHarvester.CMSHarvester.option_handler_no_ref_hists(), cmsHarvester.CMSHarvester.option_handler_ref_hist_mapping_file(), crabFunctions.CrabController.report(), crabFunctions.CrabController.resubmit(), o2olib.O2ORunMgr.startJob(), crabFunctions.CrabController.status(), and crabFunctions.CrabController.submit().

username

crabFunctions.CrabController.username

Definition at line 50 of file crabFunctions.py.

Referenced by crabFunctions.CrabController.checkusername(), and crabFunctions.CrabController.checkwrite().

voGroup

crabFunctions.CrabController.voGroup

Definition at line 46 of file crabFunctions.py.

Referenced by crabFunctions.CrabController.checkwrite().

workingArea

crabFunctions.CrabController.workingArea

Definition at line 41 of file crabFunctions.py.

Referenced by crabFunctions.CrabController.crabFolders(), and crabFunctions.CrabController.resubmit().