Function Documentation

def tools::addCrabInputFile	(	crabCfg,
		inputFile
	)

Definition at line 183 of file tools.py.

00184                                        :
00185     additionalInputFiles = ''
00186     if crabCfg.has_option('USER','additional_input_files'):
00187         additionalInputFiles = crabCfg.get('USER','additional_input_files')
00188 
00189     if additionalInputFiles: additionalInputFiles += ',%s' % inputFile
00190     else: additionalInputFiles = inputFile
00191 
00192     crabCfg.set('USER','additional_input_files',additionalInputFiles)
00193 
00194     return crabCfg

def tools::copyFilesFromCastor	(	castor_dir,
		output_dir,
		type = `'root'`
	)

Definition at line 55 of file tools.py.

00056                                                           :
00057     from subprocess import call
00058     files = listFilesInCastor(castor_dir,type,'')
00059 
00060     print "Copying files from %s to %s" % (castor_dir,output_dir) 
00061     for item in files:
00062         cmd = ['rfcp',item,output_dir] 
00063         print "..." + item
00064         retcode = call(cmd)
00065         if retcode != 0: raise RuntimeError,'Error in copying file %s to directory %s' % (item,output_dir)
00066 
00067     return 0

def tools::copyFilesLocal	(	dir,
		output_dir,
		type = `'root'`
	)

Definition at line 68 of file tools.py.

00069                                               :
00070     if not dir: raise ValueError,'Please specify valid dir'
00071     if not output_dir: raise ValueError,'Please specify valid output dir'
00072   
00073     from subprocess import call
00074     files = listFilesLocal(dir,type)
00075     cmd = ['cp']
00076     cmd.extend(files)
00077     cmd.append(output_dir)
00078     print cmd 
00079     retcode = call(cmd)
00080     return retcode

def tools::dqmWorkflowName	(	datasetpath,
		type,
		rev = `1`
	)

Definition at line 23 of file tools.py.

00024                                            :
00025     workflowName = datasetpath
00026     sections = workflowName.split('/')[1:]
00027     workflowName = '/%s/%s-%s-rev%d/%s' % (sections[0],sections[1],type,rev,sections[2])
00028     
00029     return workflowName

def tools::getDatasetStr ( datasetpath )

Definition at line 15 of file tools.py.

00016                               :
00017     datasetstr = datasetpath
00018     datasetstr.strip()
00019     if datasetstr[0] == '/': datasetstr = datasetstr[1:]
00020     datasetstr = datasetstr.replace('/','_')
00021 
00022     return datasetstr

def tools::haddInCastor	(	castor_dir,
		result_file,
		type = `'root'`,
		prefix = `'rfio:'`,
		suffix = `None`
	)

Definition at line 81 of file tools.py.

00081                                                                     :',suffix = None):
00082     if not castor_dir: raise ValueError,'Please specify valid castor dir'
00083     if not result_file: raise ValueError,'Please specify valid output file name'
00084 
00085     #cmd = 'hadd %s `./listfilesCastor %s | grep %s`'%(result_file,castor_dir,type)
00086     #print "Running",cmd
00087     #os.system(cmd)
00088     from subprocess import call
00089     files = listFilesInCastor(castor_dir,type,prefix)
00090     if suffix: files = [item + suffix for item in files]
00091  
00092     cmd = ['hadd',result_file]
00093     cmd.extend(files)
00094     #print cmd
00095     retcode = call(cmd)
00096     return retcode
00097

def tools::haddLocal	(	dir,
		result_file,
		type = `'root'`
	)

Definition at line 98 of file tools.py.

00099                                             :
00100     if not dir: raise ValueError,'Please specify valid dir'
00101     if not result_file: raise ValueError,'Please specify valid output file name'
00102 
00103     from subprocess import call
00104     files = listFilesLocal(dir,type)
00105     cmd = ['hadd',result_file]
00106     cmd.extend(files)
00107     #print cmd
00108     retcode = call(cmd)
00109     return retcode

def tools::listFilesInCastor	(	castor_dir,
		type = `'root'`,
		prefix = `'rfio:'`
	)

Definition at line 30 of file tools.py.

00030                                                              :'):
00031     if not castor_dir: raise ValueError,'Please specify valid castor dir'
00032 
00033     from subprocess import Popen,PIPE
00034     p1 = Popen(['nsls',castor_dir],stdout=PIPE)
00035     #p2 = Popen(['grep',type],stdin=p1.stdout,stdout=PIPE)
00036     #files = [prefix + castor_dir + "/" + item[:-1] for item in p2.stdout]
00037     #p2.stdout.close()
00038     files = [ "%s%s/%s" % (prefix,castor_dir,item.rstrip()) for item in p1.stdout if item.find(type) != -1 ] 
00039     p1.stdout.close()
00040     return files
00041

def tools::listFilesLocal	(	dir,
		type = `'root'`
	)

Definition at line 42 of file tools.py.

00043                                      :
00044     if not dir: raise ValueError,'Please specify valid dir'
00045 
00046     #from subprocess import Popen,PIPE
00047     #p1 = Popen(['ls',dir],stdout=PIPE)
00048     #p2 = Popen(['grep',type],stdin=p1.stdout,stdout=PIPE)
00049     #files = [dir + "/" + item[:-1] for item in p2.stdout]
00050     #p2.stdout.close()
00051     files = os.listdir(dir)
00052     files = [ "%s/%s" % (dir,item) for item in files if item.find(type) != -1 ]
00053 
00054     return files

def tools::loadCmsProcess ( psetPath )

Definition at line 144 of file tools.py.

00145                             :
00146     module = __import__(psetPath)
00147     process = sys.modules[psetPath].process
00148 
00149     import copy 
00150     #FIXME: clone process
00151     #processNew = copy.deepcopy(process)
00152     processNew = copy.copy(process) 
00153     return processNew

def tools::loadCmsProcessFile ( psetName )

Definition at line 140 of file tools.py.

00141                                 :
00142     pset = imp.load_source("psetmodule",psetName)
00143     return pset.process

def tools::loadCrabCfg ( cfgName = None )

Definition at line 178 of file tools.py.

00179                              :
00180     config = ConfigParser.ConfigParser()
00181     if cfgName: config.read(cfgName)
00182     return config

def tools::loadCrabDefault	(	crabCfg,
		config
	)

Definition at line 195 of file tools.py.

00196                                    :
00197     # CRAB section
00198     if not crabCfg.has_section('CRAB'): crabCfg.add_section('CRAB')
00199     crabCfg.set('CRAB','jobtype','cmssw')
00200 
00201     if hasattr(config,'scheduler') and config.scheduler: crabCfg.set('CRAB','scheduler',config.scheduler) 
00202     else: crabCfg.set('CRAB','scheduler','CAF')
00203 
00204     if hasattr(config,'useserver') and config.useserver: crabCfg.set('CRAB','use_server',1)
00205 
00206     # CMSSW section
00207     if not crabCfg.has_section('CMSSW'): crabCfg.add_section('CMSSW')
00208     if hasattr(config,'datasetpath') and config.datasetpath: crabCfg.set('CMSSW','datasetpath',config.datasetpath)
00209     else: crabCfg.set('CMSSW','datasetpath','/XXX/YYY/ZZZ') 
00210     crabCfg.set('CMSSW','pset','pset.py')
00211 
00212     # Splitting config
00213     crabCfg.remove_option('CMSSW','total_number_of_events')
00214     crabCfg.remove_option('CMSSW','events_per_job')
00215     crabCfg.remove_option('CMSSW','number_of_jobs')
00216     crabCfg.remove_option('CMSSW','total_number_of_lumis')
00217     crabCfg.remove_option('CMSSW','lumis_per_job')
00218     crabCfg.remove_option('CMSSW','lumi_mask')
00219     crabCfg.remove_option('CMSSW','split_by_run')
00220  
00221     """
00222     if hasattr(config,'totalnumberevents'): crabCfg.set('CMSSW','total_number_of_events',config.totalnumberevents)
00223     if hasattr(config,'eventsperjob'): crabCfg.set('CMSSW','events_per_job',config.eventsperjob) 
00224     """
00225     if hasattr(config,'splitByLumi') and config.splitByLumi:
00226         crabCfg.set('CMSSW','total_number_of_lumis',config.totalnumberlumis)
00227         crabCfg.set('CMSSW','lumis_per_job',config.lumisperjob)
00228         if hasattr(config,'lumimask') and config.lumimask: crabCfg.set('CMSSW','lumi_mask',config.lumimask)
00229     elif hasattr(config,'splitByEvent') and config.splitByEvent:
00230         crabCfg.set('CMSSW','total_number_of_events',config.totalnumberevents)
00231         crabCfg.set('CMSSW','events_per_job',config.eventsperjob)
00232     else:
00233         crabCfg.set('CMSSW','split_by_run',1)
00234 
00235     if hasattr(config,'splitByEvent') and config.splitByEvent:
00236         crabCfg.remove_option('CMSSW','runselection')
00237     else:
00238         if hasattr(config,'runselection') and config.runselection:
00239             crabCfg.set('CMSSW','runselection',config.runselection)
00240 
00241     # USER section
00242     if not crabCfg.has_section('USER'): crabCfg.add_section('USER')  
00243 
00244     # Stageout config
00245     if hasattr(config,'stageOutCAF') and config.stageOutCAF:
00246         crabCfg.set('USER','return_data',0)                
00247         crabCfg.set('USER','copy_data',1)  
00248         crabCfg.set('USER','storage_element','T2_CH_CAF')
00249         crabCfg.set('USER','user_remote_dir',config.userdircaf)
00250         crabCfg.set('USER','check_user_remote_dir',0)
00251     elif hasattr(config,'stageOutLocal') and config.stageOutLocal:
00252         crabCfg.set('USER','return_data',1)                
00253         crabCfg.set('USER','copy_data',0)
00254         crabCfg.remove_option('USER','storage_element')
00255         crabCfg.remove_option('USER','user_remote_dir')
00256         crabCfg.remove_option('USER','check_user_remote_dir')
00257 
00258     if hasattr(config,'email') and config.email: crabCfg.set('USER','eMail',config.email)
00259     crabCfg.set('USER','xml_report','crabReport.xml')
00260 
00261     if hasattr(config,'runOnGrid') and config.runOnGrid:
00262         crabCfg.remove_section('CAF')
00263         if hasattr(config,'ce_black_list'):
00264             if not crabCfg.has_section('GRID'): crabCfg.add_section('GRID')
00265             crabCfg.set('GRID','ce_black_list', config.ce_black_list)
00266         if hasattr(config,'ce_white_list'):
00267             if not crabCfg.has_section('GRID'): crabCfg.add_section('GRID')
00268             crabCfg.set('GRID','ce_white_list', config.ce_white_list)
00269     else:
00270         if not crabCfg.has_section('CAF'): crabCfg.add_section('CAF')
00271         crabCfg.set('CAF','queue',config.queueAtCAF) 
00272     
00273     return crabCfg

def tools::parseInput	(	inputFields,
		requiredFields = `()`
	)

Definition at line 122 of file tools.py.

00123                                                :
00124 
00125     class options: pass
00126     for item in sys.argv:
00127         option = item.split('=')[0]
00128         if option in inputFields:
00129             value = item.split('=')[1]
00130             if value in ('true','True','yes','Yes'): value = True
00131             elif value in ('false','False','no','No'): value = False
00132 
00133             setattr(options,option,value)
00134 
00135     for item in requiredFields:
00136         if not hasattr(options,item):
00137             raise RuntimeError,'Need to set "%s"' % item
00138 
00139     return options

def tools::prependPaths	(	process,
		seqname
	)

Definition at line 154 of file tools.py.

00155                                  :
00156     for path in process.paths: 
00157         getattr(process,path)._seq = getattr(process,seqname)*getattr(process,path)._seq

def tools::replaceTemplate	(	template,
		opts
	)

Definition at line 5 of file tools.py.

00006                                     :
00007     result = open(template).read()
00008     for item in opts:
00009          old = '@@%s@@'%item
00010          new = str(opts[item])
00011          print "Replacing",old,"to",new
00012          result = result.replace(old,new)
00013 
00014     return result

def tools::setGridEnv ( cmssw_dir )

Definition at line 110 of file tools.py.

00111                          :
00112     cwd = os.getcwd()
00113     os.chdir(cmssw_dir)
00114 
00115     os.system('source /afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh')
00116     os.system('cmsenv')
00117     os.system('source /afs/cern.ch/cms/ccs/wm/scripts/Crab/crab.sh')
00118  
00119     os.chdir(cwd)
00120  
00121     return

def tools::writeCfg	(	process,
		dir,
		psetName
	)

Definition at line 158 of file tools.py.

00159                                   :
00160     if not os.path.exists(dir): os.makedirs(dir)
00161     open(dir + '/' + psetName,'w').write(process.dumpPython())

def tools::writeCfgPkl	(	process,
		dir,
		psetName
	)

Definition at line 162 of file tools.py.

00163                                      :
00164     if not os.path.exists(dir): os.makedirs(dir)
00165 
00166     pklFileName = psetName.split('.')[0] + '.pkl'
00167     pklFile = open(dir + '/' + pklFileName,"wb")
00168     myPickle = pickle.Pickler(pklFile)
00169     myPickle.dump(process)
00170     pklFile.close()
00171  
00172     outFile = open(dir + '/' + psetName,"w")
00173     outFile.write("import FWCore.ParameterSet.Config as cms\n")
00174     outFile.write("import pickle\n")
00175     outFile.write("process = pickle.load(open('%s', 'rb'))\n" % pklFileName)
00176     outFile.close()
00177

tools Namespace Reference

Functions

Function Documentation

Functions
def	addCrabInputFile
def	copyFilesFromCastor
def	copyFilesLocal
def	dqmWorkflowName
def	getDatasetStr
def	haddInCastor
def	haddLocal
def	listFilesInCastor
def	listFilesLocal
def	loadCmsProcess
def	loadCmsProcessFile
def	loadCrabCfg
def	loadCrabDefault
def	parseInput
def	prependPaths
def	replaceTemplate
def	setGridEnv
def	writeCfg
def	writeCfgPkl