MatrixInjector.MatrixInjector Class Reference

Inheritance diagram for MatrixInjector.MatrixInjector:

Public Member Functions
def	__init__
def	prepare
def	submit
def	upload
def	uploadConf
	provide the number of tasks More...

Static Public Member Functions
def	get_wmsplit

Public Attributes
	batchName
	batchTime
	candidateWFName
	chainDicts
	couch
	couchCache
	count
	CUDACapabilities
	CUDADriverVersion
	CUDARuntime
	CUDARuntimeVersion
	DbsUrl
	defaultChain
	defaultGPUParams
	defaultHarvest
	defaultInput
	defaultScratch
	defaultTask
	dqmgui
	GPUMemoryMB
	GPUName
	group
	keep
	label
	longWFName
	memoryOffset
	memPerCore
	numberEventsInLuminosityBlock
	numberOfStreams
	RequiresGPU
	Checking and setting up GPU attributes. More...
	speciallabel
	testMode
	user
	version
	wmagent

Static Public Attributes
int	itask = 0
	the info are not in the task specific dict but in the general dict t_input.update(copy.deepcopy(self.defaultHarvest)) t_input['DQMConfigCacheID']=t_second['ConfigCacheID'] More...
list	primary = t_second['nowmIO']
list	t_input = chainDict['nowmTasklist']
list	t_second = chainDict['nowmTasklist']

Detailed Description

Definition at line 41 of file MatrixInjector.py.

Constructor & Destructor Documentation

def MatrixInjector.MatrixInjector.__init__	(		self,
			opt,
			mode = 'init',
			options = ''
	)

Definition at line 43 of file MatrixInjector.py.

    def __init__(self,opt,mode='init',options=''):
        self.count=1040

        self.dqmgui=None
        self.wmagent=None
        for k in options.split(','):
            if k.startswith('dqm:'):
                self.dqmgui=k.split(':',1)[-1]
            elif k.startswith('wma:'):
                self.wmagent=k.split(':',1)[-1]

        self.testMode=((mode!='submit') and (mode!='force'))
        self.version =1
        self.keep = opt.keep
        self.memoryOffset = opt.memoryOffset
        self.memPerCore = opt.memPerCore
        self.numberEventsInLuminosityBlock = opt.numberEventsInLuminosityBlock
        self.numberOfStreams = 0
        if(opt.nStreams>0):
            self.numberOfStreams = opt.nStreams
        self.batchName = ''
        self.batchTime = str(int(time.time()))
        if(opt.batchName):
            self.batchName = '__'+opt.batchName+'-'+self.batchTime

        ####################################
        # Checking and setting up GPU attributes
        ####################################
        # Mendatory
        self.RequiresGPU = opt.gpu
        self.GPUMemoryMB = opt.GPUMemoryMB
        self.CUDACapabilities = opt.CUDACapabilities
        self.CUDARuntime = opt.CUDARuntime
        # optional
        self.GPUName = opt.GPUName
        self.CUDADriverVersion = opt.CUDADriverVersion
        self.CUDARuntimeVersion = opt.CUDARuntimeVersion

        # WMagent url
        if not self.wmagent:
            # Overwrite with env variable
            self.wmagent = os.getenv('WMAGENT_REQMGR')

        if not self.wmagent:
            # Default values
            if not opt.testbed:
                self.wmagent = 'cmsweb.cern.ch'
            else:
                self.wmagent = 'cmsweb-testbed.cern.ch'

        # DBSReader url
        if opt.dbsUrl is not None:
            self.DbsUrl = opt.dbsUrl
        elif os.getenv('CMS_DBSREADER_URL') is not None:
            self.DbsUrl = os.getenv('CMS_DBSREADER_URL')
        else:
            # Default values
            if not opt.testbed:
                self.DbsUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
            else:
                self.DbsUrl = "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader"

        if not self.dqmgui:
            self.dqmgui="https://cmsweb.cern.ch/dqm/relval"
        #couch stuff
        self.couch = 'https://'+self.wmagent+'/couchdb'
#        self.couchDB = 'reqmgr_config_cache'
        self.couchCache={} # so that we do not upload like crazy, and recyle cfgs
        self.user = os.getenv('USER')
        self.group = 'ppd'
        self.label = 'RelValSet_'+os.getenv('CMSSW_VERSION').replace('-','')+'_v'+str(self.version)
        self.speciallabel=''
        if opt.label:
            self.speciallabel= '_'+opt.label
        self.longWFName = []

        if not os.getenv('WMCORE_ROOT'):
            print('\n\twmclient is not setup properly. Will not be able to upload or submit requests.\n')
            if not self.testMode:
                print('\n\t QUIT\n')
                sys.exit(-18)
        else:
            print('\n\tFound wmclient\n')
            
        self.defaultChain={
            "RequestType" :    "TaskChain",                    #this is how we handle relvals
            "SubRequestType" : "RelVal",                       #this is how we handle relvals, now that TaskChain is also used for central MC production
            "RequestPriority": 500000,
            "Requestor": self.user,                           #Person responsible
            "Group": self.group,                              #group for the request
            "CMSSWVersion": os.getenv('CMSSW_VERSION'),       #CMSSW Version (used for all tasks in chain)
            "Campaign": os.getenv('CMSSW_VERSION'),           # = AcquisitionEra, will be reset later to the one of first task, will both be the CMSSW_VERSION
            "ScramArch": os.getenv('SCRAM_ARCH'),             #Scram Arch (used for all tasks in chain)
            "ProcessingVersion": self.version,                #Processing Version (used for all tasks in chain)
            "GlobalTag": None,                                #Global Tag (overridden per task)
            "ConfigCacheUrl": self.couch,                     #URL of CouchDB containing Config Cache
            "DbsUrl": self.DbsUrl,
            #- Will contain all configs for all Tasks
            #"SiteWhitelist" : ["T2_CH_CERN", "T1_US_FNAL"],   #Site whitelist
            "TaskChain" : None,                                  #Define number of tasks in chain.
            "nowmTasklist" : [],  #a list of tasks as we put them in
            "Multicore" : 1,   # do not set multicore for the whole chain
            "Memory" : 3000,
            "SizePerEvent" : 1234,
            "TimePerEvent" : 10,
            "PrepID": os.getenv('CMSSW_VERSION')
            }

        self.defaultHarvest={
            "EnableHarvesting" : "True",
            "DQMUploadUrl" : self.dqmgui,
            "DQMConfigCacheID" : None,
            "Multicore" : 1              # hardcode Multicore to be 1 for Harvest
            }
        
        self.defaultScratch={
            "TaskName" : None,                            #Task Name
            "ConfigCacheID" : None,                   #Generator Config id
            "GlobalTag": None,
            "SplittingAlgo"  : "EventBased",             #Splitting Algorithm
            "EventsPerJob" : None,                       #Size of jobs in terms of splitting algorithm
            "EventsPerLumi" : None,
            "RequestNumEvents" : None,                      #Total number of events to generate
            "Seeding" : "AutomaticSeeding",                          #Random seeding method
            "PrimaryDataset" : None,                          #Primary Dataset to be created
            "nowmIO": {},
            "Multicore" : opt.nThreads,                  # this is the per-taskchain Multicore; it's the default assigned to a task if it has no value specified 
            "EventStreams": self.numberOfStreams,
            "KeepOutput" : False
            }
        self.defaultInput={
            "TaskName" : "DigiHLT",                                      #Task Name
            "ConfigCacheID" : None,                                      #Processing Config id
            "GlobalTag": None,
            "InputDataset" : None,                                       #Input Dataset to be processed
            "SplittingAlgo"  : "LumiBased",                        #Splitting Algorithm
            "LumisPerJob" : 10,               #Size of jobs in terms of splitting algorithm
            "nowmIO": {},
            "Multicore" : opt.nThreads,                       # this is the per-taskchain Multicore; it's the default assigned to a task if it has no value specified 
            "EventStreams": self.numberOfStreams,
            "KeepOutput" : False
            }
        self.defaultTask={
            "TaskName" : None,                                 #Task Name
            "InputTask" : None,                                #Input Task Name (Task Name field of a previous Task entry)
            "InputFromOutputModule" : None,                    #OutputModule name in the input task that will provide files to process
            "ConfigCacheID" : None,                            #Processing Config id
            "GlobalTag": None,
            "SplittingAlgo"  : "LumiBased",                        #Splitting Algorithm
            "LumisPerJob" : 10,               #Size of jobs in terms of splitting algorithm
            "nowmIO": {},
            "Multicore" : opt.nThreads,                       # this is the per-taskchain Multicore; it's the default assigned to a task if it has no value specified 
            "EventStreams": self.numberOfStreams,
            "KeepOutput" : False,
            "RequiresGPU" : None,
            "GPUParams": None
            }
        self.defaultGPUParams={
            "GPUMemoryMB": self.GPUMemoryMB,
            "CUDACapabilities": self.CUDACapabilities,
            "CUDARuntime": self.CUDARuntime
            }
        if self.GPUName: self.defaultGPUParams.update({"GPUName": self.GPUName})
        if self.CUDADriverVersion: self.defaultGPUParams.update({"CUDADriverVersion": self.CUDADriverVersion})
        if self.CUDARuntimeVersion: self.defaultGPUParams.update({"CUDARuntimeVersion": self.CUDARuntimeVersion})

        self.chainDicts={}