Functions
def	append_sample_mcrab (dsetp, fout)

def	check_dbs ()

def	check_dset ()

def	check_nevts_dset (dset)

def	create_crab (ds)

def	create_harvest (ds)

def	create_mcrab (set, fcrab, fout)

def	find_dqmref (ds)

def	get_cond_from_dsetpath (ds)

def	get_name_from_dsetpath (ds)

def	make_dbs_list (dbslf)

def	make_dqmname (s)

def	make_harv_name (dset)

def	print_def ()

def	read_ds_file ()

Variables
	api

	argin

	args

	cmssw_ver

	crab_block

	do_reference

	dsetpaths

	dsfile

	dslproc

	f_crab
	Create harvest.py template. More...

	f_multi_crab
	Create harvest_n.py for individual datasets. More...

	harvfilelist
	Print what has been created. More...

	input_type

	is_dbs

	optManager

	opts

Function Documentation

◆ append_sample_mcrab()

def harvestRelVal.append_sample_mcrab	(	dsetp,
		fout
	)

Definition at line 215 of file harvestRelVal.py.

References find_dqmref(), get_name_from_dsetpath(), make_dqmname(), make_harv_name(), and print().

Referenced by create_mcrab().

 def append_sample_mcrab(dsetp, fout):
     dqm = make_dqmname(dsetp)
     sample = get_name_from_dsetpath(dsetp)
     hf = make_harv_name(dsetp)
     if not os.path.exists(hf) :
         print('problem creating multicrab, file', hf, 'does not exist')
         sys.exit(17)
     fout.write('\n\n[' + sample + ']')
     fout.write('\nCMSSW.pset=' + hf)
     fout.write('\nCMSSW.datasetpath=' + dsetp)
     fout.write('\nCMSSW.output_file=' + dqm)
 
     dqmref = find_dqmref(dsetp);
     if not dqmref == 'NONE' : 
         fout.write('\nUSER.additional_input_files=' + dqmref)
 

◆ check_dbs()

def harvestRelVal.check_dbs ( )

Definition at line 25 of file harvestRelVal.py.

 def check_dbs():
     if os.getenv('DBSCMD_HOME','NOTSET') == 'NOTSET' :
         return 0
     return 1
 

◆ check_dset()

def harvestRelVal.check_dset ( )

Definition at line 97 of file harvestRelVal.py.

References check_nevts_dset(), get_cond_from_dsetpath(), and print().

 def check_dset() :
    #check cmssw consistency
    for s in dsetpaths:
        if s.find(cmssw_ver) == -1 :
            dsetpaths.remove(s)        
            print('Inconsistency found with datset and cmssw version (', cmssw_ver, ')' \
                  ': \t ', s, ' has been removed.')
    #check conditions from dsetname
    for s in dsetpaths[:]: #nb:need to make a copy here!
        cond = get_cond_from_dsetpath(s)
        if cond  == 0 : 
            dsetpaths.remove(s)        
    #check list size
    nSamples = len(dsetpaths)
    if nSamples == 0 :
        print("Empty input list, exit.")
        sys.exit(12)
    else :
        print('Processing', nSamples, 'data sets.')
    #check event numbers
    nSampleEvts = list()
    for s in dsetpaths:
        nSampleEvts.append(check_nevts_dset(s))
    print('number of events per dataset:', nSampleEvts)
 

◆ check_nevts_dset()

def harvestRelVal.check_nevts_dset ( dset )

Definition at line 30 of file harvestRelVal.py.

References str.

Referenced by check_dset().

 def check_nevts_dset(dset):
     if not is_dbs :
         return -1
     ntot=0
     for afile in api.listFiles(path=str(dset)):
         nevts = afile['NumberOfEvents']
         ntot += nevts
         #print "  %s" % afile['LogicalFileName']
     return ntot  
 

◆ create_crab()

def harvestRelVal.create_crab ( ds )

Definition at line 231 of file harvestRelVal.py.

References make_dqmname(), and make_harv_name().

 def create_crab(ds) :
     dqmout = make_dqmname(ds)
     hf = make_harv_name(ds)
     out = open(f_crab, 'w')
     out.write(crab_block)
     out.write('\npset=' + hf)
     out.write('datasetpath=' + ds)
     out.write('\noutput_file=' + dqmout)
     out.close()
 

◆ create_harvest()

def harvestRelVal.create_harvest ( ds )

Definition at line 156 of file harvestRelVal.py.

References find_dqmref(), get_cond_from_dsetpath(), make_harv_name(), and print().

 def create_harvest(ds) :
     raw_cmsdriver = "cmsDriver.py harvest -s HARVESTING:validationHarvesting --mc  --conditions FrontierConditions_GlobalTag,STARTUP_30X::All --harvesting AtJobEnd --no_exec -n -1"
     cmsdriver = raw_cmsdriver
     cond = get_cond_from_dsetpath(ds)
     if cond == 0 :
         print('unexpected problem with conditions')
         sys.exit(50)
     cmsdriver = cmsdriver.replace('30X',cond)
     fin_name="harvest_HARVESTING_STARTUP.py"
     if ds.find('IDEAL') != -1 :
         cmsdriver = cmsdriver.replace('STARTUP','IDEAL')
         fin_name = fin_name.replace('STARTUP','IDEAL')
     if ds.find('FastSim') != -1:
         cmsdriver = cmsdriver.replace('validationHarvesting','validationHarvestingFS')
     if ds.find('PileUp') != -1:
         cmsdriver = cmsdriver.replace('validationHarvesting','validationHarvestingPU')
 
     #print "=>", cmsdriver, " fs?", ds.find('FastSim')
     if os.path.exists(fin_name) : 
         os.system("rm " + fin_name)
     print("executing cmsdriver command:\n\t", cmsdriver)
     os.system(cmsdriver)
     if not os.path.exists(fin_name) : 
         print('problem with cmsdriver file name')
         sys.exit(40)
     os.system("touch " + fin_name)
     hf = make_harv_name(ds)
     os.system('mv ' + fin_name + " " + hf)
     out = open(hf, 'a')
     out.write("\n\n##additions to cmsDriver output \n")
     out.write("process.dqmSaver.workflow = '" + ds + "'\n")
     if is_dbs :
         out.write("process.source.fileNames = cms.untracked.vstring(\n")
         for afile in api.listFiles(path=ds):
             out.write("  '%s',\n" % afile['LogicalFileName'])
         out.write(")\n")
 
     dqmref = find_dqmref(ds);
     if not dqmref == 'NONE' : 
         out.write("process.dqmSaver.referenceHandling = 'all'\n")
 
     out.close()
 

◆ create_mcrab()

def harvestRelVal.create_mcrab	(	set,
		fcrab,
		fout
	)

Definition at line 199 of file harvestRelVal.py.

References append_sample_mcrab().

 def create_mcrab(set, fcrab, fout):
     out = open(fout, 'w')
     out.write('[MULTICRAB]')
     out.write('\ncfg=' + fcrab)
     out.write('\n\n[COMMON]')
     nevt = -1
     njob = 1
     out.write('\nCMSSW.total_number_of_events=' + (str)(nevt) )
     out.write('\nCMSSW.number_of_jobs=' + (str)(njob) )
     for s in set:
         append_sample_mcrab(s, out)
     out.close()    
 

◆ find_dqmref()

def harvestRelVal.find_dqmref ( ds )

Definition at line 122 of file harvestRelVal.py.

References make_dqmname(), print(), python.rootplot.root2matplotlib.replace(), and str.

Referenced by append_sample_mcrab(), and create_harvest().

 def find_dqmref(ds) :
     if not do_reference :
         return 'NONE'
     cp = cmssw_ver[-1:]
     ip = (int)(cp) - 1
     ref_ver = cmssw_ver.replace(cp,str(ip))
     #print "cms:", cmssw_ver, " cp:", cp, " ip:", ip, " new_ver:", ref_ver  
     ref_dir = "/castor/cern.ch/user/n/nuno/relval/harvest/" + ref_ver + "/"
     ref_dsf = make_dqmname(ds.replace(cmssw_ver, ref_ver))
     gls = " | grep root | grep "
     #to accept crab appended _1.root in file names, nd skip versions/conditions
     gls += ref_dsf[:-25] 
     gls += "| awk '{print $9}' "
     #print "refds:", ref_dsf, " command: rfdir", ref_dir+gls
     command = "rfcp " + ref_dir  + "`rfdir " + ref_dir + gls + "` ."
     #print "command:", command
     os.system(command)
     tmpfile = "ref.txt"
     command = "ls -rtl *" + gls + " > " + tmpfile
     #print "command:", command
     os.system(command)
     the_ref = 'NONE'
     if os.path.exists(tmpfile) :
         fin = open(tmpfile,'r')        ref = fin.readline().replace('\n','')
         #print "read ref:", ref, "exists?", os.path.exists(ref)
         fin.close()
         if os.path.exists(ref) :
             the_ref = ref
     else :
         the_ref = 'NONE'
     print("Found reference file:", the_ref)
     return the_ref
 

◆ get_cond_from_dsetpath()

def harvestRelVal.get_cond_from_dsetpath ( ds )

Definition at line 48 of file harvestRelVal.py.

References spr.find(), print(), and python.rootplot.root2matplotlib.replace().

Referenced by check_dset(), and create_harvest().

 def get_cond_from_dsetpath(ds) :
     ca = ds.split('/')[2].replace(cmssw_ver+'_','').replace('IDEAL_','').replace('STARTUP_','').replace('_FastSim','')
     cb = ca[:ca.find('v')-1]
     if cb[0].find('3') == -1 or len(cb) > 3:
         print("problem extracting condition for", ds, " : ", cb, '(len:',len(cb),')')  
         if cb.find('31X') != -1:
             cb = '31X'
         elif cb.find('30X') != -1:
             cb = '30X'
         else:
             print("skipping", cb)
             return 0
         print("condition found:", cb)
     else :
         print("good condition for", ds, " : ", cb, '(len:',len(cb),')')      
     return cb
 
 

◆ get_name_from_dsetpath()

def harvestRelVal.get_name_from_dsetpath ( ds )

Definition at line 43 of file harvestRelVal.py.

References python.rootplot.root2matplotlib.replace().

Referenced by append_sample_mcrab(), and make_harv_name().

 def get_name_from_dsetpath(ds):
     fs = ds.split('/')
     fa = fs[1].replace('RelVal','')
     return fa
 

◆ make_dbs_list()

def harvestRelVal.make_dbs_list ( dbslf )

Definition at line 66 of file harvestRelVal.py.

References print().

 def make_dbs_list(dbslf) :
     if not is_dbs :
         return
     flis = open(dbslf,'w')
     for ads in api.listDatasetPaths() :
         if ads.find('RelVal') != -1 \
                or ads.find(cmssw_ver) != -1 \
                or ads.find("/GEN-SIM") != -1 : 
 #               and ads.find("/GEN-SIM-RECO") != -1 : 
             flis.write(ads + '\n')
     flis.close()
     print('Generated dataset list', dbslf, 'from dbs.') 
     #exampe:
     #dbs lsd --path=/RelVal*/CMSSW_3_1_0_pre5*/GEN-SIM-RECO --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet > mylist.txt
     #dbslsd = "dbs lsd --path=/RelVal*/" + cmssw_ver + "*/GEN-SIM-RECO --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
     #os.system( '`' + dbslsd + ' > ' + dbslf + '`')
 

◆ make_dqmname()

def harvestRelVal.make_dqmname ( s )

Definition at line 40 of file harvestRelVal.py.

Referenced by append_sample_mcrab(), create_crab(), and find_dqmref().

 def make_dqmname(s):
     return  'DQM_V0001_R000000001' + s.replace('/','__') + '.root' 
 

◆ make_harv_name()

def harvestRelVal.make_harv_name ( dset )

Definition at line 212 of file harvestRelVal.py.

References get_name_from_dsetpath().

Referenced by append_sample_mcrab(), create_crab(), and create_harvest().

 def make_harv_name(dset) :
     return 'harvest_' + get_name_from_dsetpath(dset) + '.py' 
 

◆ print_def()

def harvestRelVal.print_def ( )

Definition at line 18 of file harvestRelVal.py.

References print().

 def print_def():
     print("Usage:", sys.argv[0], "[list_of_processes]")
     print("Examples:")
     print("harvestRelVal.py")
     print("harvestRelVal.py /RelValTTbar/CMSSW_3_1_0_pre4_STARTUP_30X_v1/GEN-SIM-RECO")
     print("harvestRelVal.py <dataset_list.txt>")
 

◆ read_ds_file()

def harvestRelVal.read_ds_file ( )

Definition at line 83 of file harvestRelVal.py.

References print().

 def read_ds_file() :
     if not os.path.exists(dsfile) :
         print("problem reading file", dsfile)
         sys.exit(30)
     fin = open(dsfile,'r')
     for dset in fin.readlines(): 
         d = dset.replace('\n','')
         if d.find('#') == -1 :
             dsetpaths.append(d)
         else :
             print('skipping:', d)
     fin.close()
     print('Using data set list in ', dsfile)
 

Variable Documentation

◆ api

harvestRelVal.api

Definition at line 307 of file harvestRelVal.py.

◆ argin

harvestRelVal.argin

Definition at line 269 of file harvestRelVal.py.

◆ args

harvestRelVal.args

Definition at line 303 of file harvestRelVal.py.

◆ cmssw_ver

harvestRelVal.cmssw_ver

Definition at line 310 of file harvestRelVal.py.

◆ crab_block

harvestRelVal.crab_block

Definition at line 241 of file harvestRelVal.py.

◆ do_reference

harvestRelVal.do_reference

Definition at line 271 of file harvestRelVal.py.

◆ dsetpaths

harvestRelVal.dsetpaths

Definition at line 326 of file harvestRelVal.py.

◆ dsfile

harvestRelVal.dsfile

Definition at line 270 of file harvestRelVal.py.

◆ dslproc

harvestRelVal.dslproc

Definition at line 348 of file harvestRelVal.py.

◆ f_crab

harvestRelVal.f_crab

Create harvest.py template.

Create crab.cfg template

Definition at line 358 of file harvestRelVal.py.

◆ f_multi_crab

harvestRelVal.f_multi_crab

Create harvest_n.py for individual datasets.

Create multicrab.cfg

Definition at line 366 of file harvestRelVal.py.

◆ harvfilelist

harvestRelVal.harvfilelist

Print what has been created.

Definition at line 371 of file harvestRelVal.py.

◆ input_type

harvestRelVal.input_type

Definition at line 268 of file harvestRelVal.py.

◆ is_dbs

harvestRelVal.is_dbs

Definition at line 293 of file harvestRelVal.py.

◆ optManager

harvestRelVal.optManager

Definition at line 302 of file harvestRelVal.py.

◆ opts

harvestRelVal.opts

Definition at line 303 of file harvestRelVal.py.

Functions

Variables

Function Documentation

◆ append_sample_mcrab()

◆ check_dbs()

◆ check_dset()

◆ check_nevts_dset()

◆ create_crab()

◆ create_harvest()

◆ create_mcrab()

◆ find_dqmref()

◆ get_cond_from_dsetpath()

◆ get_name_from_dsetpath()

◆ make_dbs_list()

◆ make_dqmname()

◆ make_harv_name()

◆ print_def()

◆ read_ds_file()

Variable Documentation

◆ api

◆ argin

◆ args

◆ cmssw_ver

◆ crab_block

◆ do_reference

◆ dsetpaths

◆ dsfile

◆ dslproc

◆ f_crab

◆ f_multi_crab

◆ harvfilelist

◆ input_type

◆ is_dbs

◆ optManager

◆ opts